From 343526ef4166a429098cf96dbb48a688d966a917 Mon Sep 17 00:00:00 2001 From: "fisher.yu" Date: Wed, 6 Nov 2024 12:05:57 +0000 Subject: [PATCH 1/3] [lib]: code refactoring in aarch64 library - update multi-buffer manager code for aarch64 platforms Signed-off-by: Islam Ragimov --- lib/aarch64/mb_mgr_aarch64.c | 141 +----------------------- lib/aarch64/mb_mgr_aarch64.h | 147 ++++++++++++++++++++++++++ lib/aarch64/mb_mgr_aarch64_no_aesni.c | 133 +---------------------- lib/aarch64/mb_mgr_aarch64_sve256.c | 143 +------------------------ lib/include/arch_aarch64.h | 107 +++++++++++++++---- lib/include/arch_aarch64_noaesni.h | 85 ++++++++++----- lib/include/arch_aarch64_sve256.h | 85 ++++++++++----- lib/include/ipsec_ooo_mgr.h | 6 +- lib/x86_64/alloc.c | 4 +- 9 files changed, 356 insertions(+), 495 deletions(-) create mode 100644 lib/aarch64/mb_mgr_aarch64.h diff --git a/lib/aarch64/mb_mgr_aarch64.c b/lib/aarch64/mb_mgr_aarch64.c index 5efe2018..937d767d 100644 --- a/lib/aarch64/mb_mgr_aarch64.c +++ b/lib/aarch64/mb_mgr_aarch64.c @@ -26,145 +26,6 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ -#include -#include -#include - -#include "ipsec-mb.h" -#include "include/snow3g.h" -#include "include/zuc_internal.h" - -#include "include/cpu_feature.h" -#include "include/error.h" -#include "clear_regs_mem_aarch64.h" -#include "include/noaesni.h" -#include "include/ipsec_ooo_mgr.h" #include "arch_aarch64.h" -#include "include/ooo_mgr_reset.h" - -#define SUBMIT_JOB submit_job_aarch64 -#define FLUSH_JOB flush_job_aarch64 -#define QUEUE_SIZE queue_size_aarch64 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64 -#define GET_NEXT_JOB get_next_job_aarch64 -#define GET_COMPLETED_JOB get_completed_job_aarch64 -#define GET_NEXT_BURST get_next_burst_aarch64 -#define SUBMIT_BURST submit_burst_aarch64 -#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64 -#define FLUSH_BURST flush_burst_aarch64 -#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64 -#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64 -#define SUBMIT_HASH_BURST submit_hash_burst_aarch64 -#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64 -#define SET_SUITE_ID_FN set_suite_id_aarch64 - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64 -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64 -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64 -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64 -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64 -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64 -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64 -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64 -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64 - -static void reset_ooo_mgrs(IMB_MGR *state) -{ - /* Init ZUC out-of-order fields */ - ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); - - /* Init SNOW3G-UEA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); - - /* Init SNOW3G-UIA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); -} - -IMB_DLL_LOCAL void -init_mb_mgr_aarch64_internal(IMB_MGR *state, const int reset_mgrs) -{ -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_AARCH64; - - if (!(state->features & IMB_FEATURE_AESNI)) { - init_mb_mgr_aarch64_no_aesni(state); - return; - } - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set AARCH64 handlers */ - state->get_next_job = get_next_job_aarch64; - state->submit_job = submit_job_aarch64; - state->submit_job_nocheck = submit_job_nocheck_aarch64; - state->get_completed_job = get_completed_job_aarch64; - state->flush_job = flush_job_aarch64; - state->queue_size = queue_size_aarch64; - state->get_next_burst = GET_NEXT_BURST; - state->submit_burst = SUBMIT_BURST; - state->submit_burst_nocheck = SUBMIT_BURST_NOCHECK; - state->flush_burst = FLUSH_BURST; - state->submit_cipher_burst = SUBMIT_CIPHER_BURST; - state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; - state->submit_hash_burst = SUBMIT_HASH_BURST; - state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; - state->set_suite_id = SET_SUITE_ID_FN; - - state->eea3_1_buffer = zuc_eea3_1_buffer_aarch64; - state->eea3_4_buffer = zuc_eea3_4_buffer_aarch64; - state->eea3_n_buffer = zuc_eea3_n_buffer_aarch64; - state->zuc256_eea3_1_buffer = zuc256_eea3_1_buffer_aarch64; - state->zuc256_eea3_n_buffer = zuc256_eea3_n_buffer_aarch64; - state->eia3_1_buffer = zuc_eia3_1_buffer_aarch64; - state->eia3_n_buffer = zuc_eia3_n_buffer_aarch64; - state->zuc256_eia3_1_buffer = zuc256_eia3_1_buffer_aarch64; - state->zuc256_eia3_n_buffer = zuc256_eia3_n_buffer_aarch64; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_aarch64; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_aarch64; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_aarch64; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_aarch64; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_aarch64; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_aarch64; - state->snow3g_f8_4_buffer_multikey = snow3g_f8_4_buffer_multikey_aarch64; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_aarch64; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_aarch64; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_aarch64; - state->snow3g_init_key_sched = snow3g_init_key_sched_aarch64; - state->snow3g_key_sched_size = snow3g_key_sched_size_aarch64; - - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_aarch64; -} - -void -init_mb_mgr_aarch64(IMB_MGR *state) -{ - init_mb_mgr_aarch64_internal(state, 1); -} -#include "mb_mgr_code.h" +#include "mb_mgr_aarch64.h" diff --git a/lib/aarch64/mb_mgr_aarch64.h b/lib/aarch64/mb_mgr_aarch64.h new file mode 100644 index 00000000..0e7ccda7 --- /dev/null +++ b/lib/aarch64/mb_mgr_aarch64.h @@ -0,0 +1,147 @@ +/********************************************************************** + SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its + affiliates + SPDX-License-Identifier: BSD-3-Clause + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ +#include +#include +#include + +#include "ipsec-mb.h" +#include "include/snow3g.h" +#include "include/zuc_internal.h" + +#include "include/cpu_feature.h" +#include "include/error.h" +#include "clear_regs_mem_aarch64.h" +#include "include/noaesni.h" +#include "include/ipsec_ooo_mgr.h" + +#include "include/ooo_mgr_reset.h" + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, NUM_ZUC_LANES); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, NUM_ZUC_LANES); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, NUM_ZUC_LANES); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, NUM_ZUC_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, NUM_SNOW3G_LANES); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, NUM_SNOW3G_LANES); +} + +IMB_DLL_LOCAL void +INIT_MB_MGR_INTERNAL(IMB_MGR *state, const int reset_mgrs) +{ +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return; + } +#endif + + /* reset error status */ + imb_set_errno(state, 0); + + state->features = cpu_feature_adjust(state->flags, + cpu_feature_detect()); + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) USED_ARCH; + + if (!(state->features & IMB_FEATURE_AESNI) + && (USED_ARCH != IMB_ARCH_NOAESNI)) { + init_mb_mgr_aarch64_no_aesni(state); + return; + } + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set AARCH64 handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck = SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; + + + state->eea3_1_buffer = ZUC_EEA3_1_BUFFER; + state->eea3_4_buffer = ZUC_EEA3_4_BUFFER; + state->eea3_n_buffer = ZUC_EEA3_N_BUFFER; + state->zuc256_eea3_1_buffer = ZUC256_EEA_1_BUFFER; + state->zuc256_eea3_n_buffer = ZUC256_EEA_N_BUFFER; + state->eia3_1_buffer = ZUC_EIA_1_BUFFER; + state->eia3_n_buffer = ZUC_EIA_N_BUFFER; + state->zuc256_eia3_1_buffer = ZUC256_EIA3_1_BUFFER; + state->zuc256_eia3_n_buffer = ZUC256_EIA3_N_BUFFER; + + state->snow3g_f8_1_buffer_bit = SNOW3G_F8_1_BUFFER_BIT; + state->snow3g_f8_1_buffer = SNOW3G_F8_1_BUFFER; + state->snow3g_f8_2_buffer = SNOW3G_F8_2_BUFFER; + state->snow3g_f8_4_buffer = SNOW3G_F8_4_BUFFER; + state->snow3g_f8_8_buffer = SNOW3G_F8_8_BUFFER; + state->snow3g_f8_n_buffer = SNOW3G_F8_N_BUFFER; + state->snow3g_f8_4_buffer_multikey = SNOW3G_F8_4_BUFFER_MULTIKEY; + state->snow3g_f8_8_buffer_multikey = SNOW3G_F8_8_BUFFER_MULTIKEY; + state->snow3g_f8_n_buffer_multikey = SNOW3G_F8_N_BUFFER_MULTIKEY; + state->snow3g_f9_1_buffer = SNOW3G_F9_1_BUFFER; + state->snow3g_init_key_sched = SNOW3G_INIT_KEY_SCHED; + state->snow3g_key_sched_size = SNOW3G_KEY_SCHED_SIZE; + + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_aarch64; +} + +void +INIT_MB_MGR(IMB_MGR *state) +{ + INIT_MB_MGR_INTERNAL(state, 1); +} +#include "mb_mgr_code.h" diff --git a/lib/aarch64/mb_mgr_aarch64_no_aesni.c b/lib/aarch64/mb_mgr_aarch64_no_aesni.c index df4932c3..39a2b234 100644 --- a/lib/aarch64/mb_mgr_aarch64_no_aesni.c +++ b/lib/aarch64/mb_mgr_aarch64_no_aesni.c @@ -26,137 +26,6 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ -#include -#include -#include - -#include "ipsec-mb.h" -#include "include/snow3g.h" -#include "include/zuc_internal.h" - -#include "include/noaesni.h" -#include "include/error.h" -#include "include/ipsec_ooo_mgr.h" #include "arch_aarch64_noaesni.h" -#include "include/ooo_mgr_reset.h" - -#define SUBMIT_JOB submit_job_aarch64_no_aesni -#define FLUSH_JOB flush_job_aarch64_no_aesni -#define QUEUE_SIZE queue_size_aarch64_no_aesni -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_no_aesni -#define GET_NEXT_JOB get_next_job_aarch64_no_aesni -#define GET_COMPLETED_JOB get_completed_job_aarch64_no_aesni -#define GET_NEXT_BURST get_next_burst_aarch64_no_aesni -#define SUBMIT_BURST submit_burst_aarch64_no_aesni -#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64_no_aesni -#define FLUSH_BURST flush_burst_aarch64_no_aesni -#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64_no_aesni -#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64_no_aesni -#define SUBMIT_HASH_BURST submit_hash_burst_aarch64_no_aesni -#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_no_aesni -#define SET_SUITE_ID_FN set_suite_id_aarch64_no_aesni - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_no_aesni -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_no_aesni -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64_no_aesni -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64_no_aesni -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64_no_aesni -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64_no_aesni -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64_no_aesni -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64_no_aesni -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_no_aesni -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_no_aesni -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_no_aesni -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_no_aesni - -static void reset_ooo_mgrs(IMB_MGR *state) -{ - /* Init ZUC out-of-order fields */ - ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); - - /* Init SNOW3G-UEA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); - - /* Init SNOW3G-UIA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); -} - -IMB_DLL_LOCAL void -init_mb_mgr_aarch64_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) -{ -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - imb_set_errno(state, 0); - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_NOAESNI; - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set AARCH64 NO AESNI handlers */ - state->get_next_job = get_next_job_aarch64_no_aesni; - state->submit_job = submit_job_aarch64_no_aesni; - state->submit_job_nocheck = submit_job_nocheck_aarch64_no_aesni; - state->get_completed_job = get_completed_job_aarch64_no_aesni; - state->flush_job = flush_job_aarch64_no_aesni; - state->queue_size = queue_size_aarch64_no_aesni; - state->get_next_burst = GET_NEXT_BURST; - state->submit_burst = SUBMIT_BURST; - state->submit_burst_nocheck = SUBMIT_BURST_NOCHECK; - state->flush_burst = FLUSH_BURST; - state->submit_cipher_burst = SUBMIT_CIPHER_BURST; - state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; - state->submit_hash_burst = SUBMIT_HASH_BURST; - state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; - state->set_suite_id = SET_SUITE_ID_FN; - - state->eea3_1_buffer = zuc_eea3_1_buffer_aarch64_no_aesni; - state->eea3_4_buffer = zuc_eea3_4_buffer_aarch64_no_aesni; - state->eea3_n_buffer = zuc_eea3_n_buffer_aarch64_no_aesni; - state->zuc256_eea3_1_buffer = zuc256_eea3_1_buffer_aarch64_no_aesni; - state->zuc256_eea3_n_buffer = zuc256_eea3_n_buffer_aarch64_no_aesni; - state->eia3_1_buffer = zuc_eia3_1_buffer_aarch64_no_aesni; - state->eia3_n_buffer = zuc_eia3_n_buffer_aarch64_no_aesni; - state->zuc256_eia3_1_buffer = zuc256_eia3_1_buffer_aarch64_no_aesni; - state->zuc256_eia3_n_buffer = zuc256_eia3_n_buffer_aarch64_no_aesni; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_aarch64_no_aesni; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_aarch64_no_aesni; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_aarch64_no_aesni; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_aarch64_no_aesni; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_aarch64_no_aesni; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_aarch64_no_aesni; - state->snow3g_f8_4_buffer_multikey = - snow3g_f8_4_buffer_multikey_aarch64_no_aesni; - state->snow3g_f8_8_buffer_multikey = - snow3g_f8_8_buffer_multikey_aarch64_no_aesni; - state->snow3g_f8_n_buffer_multikey = - snow3g_f8_n_buffer_multikey_aarch64_no_aesni; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_aarch64_no_aesni; - state->snow3g_init_key_sched = snow3g_init_key_sched_aarch64_no_aesni; - state->snow3g_key_sched_size = snow3g_key_sched_size_aarch64_no_aesni; - - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_aarch64; -} - -void -init_mb_mgr_aarch64_no_aesni(IMB_MGR *state) -{ - init_mb_mgr_aarch64_no_aesni_internal(state, 1); -} -#include "mb_mgr_code.h" +#include "mb_mgr_aarch64.h" diff --git a/lib/aarch64/mb_mgr_aarch64_sve256.c b/lib/aarch64/mb_mgr_aarch64_sve256.c index 19dc6bb2..a631aa0e 100644 --- a/lib/aarch64/mb_mgr_aarch64_sve256.c +++ b/lib/aarch64/mb_mgr_aarch64_sve256.c @@ -26,147 +26,6 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ -#include -#include -#include - -#include "ipsec-mb.h" -#include "include/snow3g.h" -#include "include/zuc_internal.h" - -#include "include/cpu_feature.h" -#include "include/error.h" -#include "clear_regs_mem_aarch64.h" -#include "include/noaesni.h" -#include "include/ipsec_ooo_mgr.h" #include "arch_aarch64_sve256.h" -#include "arch_aarch64.h" - -#include "include/ooo_mgr_reset.h" - -#define SUBMIT_JOB submit_job_aarch64_sve256 -#define FLUSH_JOB flush_job_aarch64_sve256 -#define QUEUE_SIZE queue_size_aarch64_sve256 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_sve256 -#define GET_NEXT_JOB get_next_job_aarch64_sve256 -#define GET_COMPLETED_JOB get_completed_job_aarch64_sve256 -#define GET_NEXT_BURST get_next_burst_aarch64_sve256 -#define SUBMIT_BURST submit_burst_aarch64_sve256 -#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64_sve256 -#define FLUSH_BURST flush_burst_aarch64_sve256 -#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64_sve256 -#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64_sve256 -#define SUBMIT_HASH_BURST submit_hash_burst_aarch64_sve256 -#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_sve256 -#define SET_SUITE_ID_FN set_suite_id_aarch64_sve256 - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64 -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64 -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64 -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64 -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64 -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_sve256 -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_sve256 -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_sve256 -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_sve256 - -static void reset_ooo_mgrs(IMB_MGR *state) -{ - /* Init ZUC out-of-order fields */ - ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); - - /* Init SNOW3G-UEA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 8); - - /* Init SNOW3G-UIA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 8); -} - -IMB_DLL_LOCAL void -init_mb_mgr_aarch64_sve256_internal(IMB_MGR *state, const int reset_mgrs) -{ -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_SVE256; - - if (!(state->features & IMB_FEATURE_AESNI)) { - init_mb_mgr_aarch64_no_aesni(state); - return; - } - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set AARCH64 handlers */ - state->get_next_job = get_next_job_aarch64_sve256; - state->submit_job = submit_job_aarch64_sve256; - state->submit_job_nocheck = submit_job_nocheck_aarch64_sve256; - state->get_completed_job = get_completed_job_aarch64_sve256; - state->flush_job = flush_job_aarch64_sve256; - state->queue_size = queue_size_aarch64_sve256; - state->get_next_burst = GET_NEXT_BURST; - state->submit_burst = SUBMIT_BURST; - state->submit_burst_nocheck = SUBMIT_BURST_NOCHECK; - state->flush_burst = FLUSH_BURST; - state->submit_cipher_burst = SUBMIT_CIPHER_BURST; - state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; - state->submit_hash_burst = SUBMIT_HASH_BURST; - state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; - state->set_suite_id = SET_SUITE_ID_FN; - - state->eea3_1_buffer = zuc_eea3_1_buffer_aarch64; - state->eea3_4_buffer = zuc_eea3_4_buffer_aarch64; - state->eea3_n_buffer = zuc_eea3_n_buffer_aarch64; - state->zuc256_eea3_1_buffer = zuc256_eea3_1_buffer_aarch64; - state->zuc256_eea3_n_buffer = zuc256_eea3_n_buffer_aarch64; - state->eia3_1_buffer = zuc_eia3_1_buffer_aarch64; - state->eia3_n_buffer = zuc_eia3_n_buffer_aarch64; - state->zuc256_eia3_1_buffer = zuc256_eia3_1_buffer_aarch64; - state->zuc256_eia3_n_buffer = zuc256_eia3_n_buffer_aarch64; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_aarch64_sve256; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_aarch64_sve256; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_aarch64_sve256; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_aarch64_sve256; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_aarch64_sve256; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_aarch64_sve256; - state->snow3g_f8_4_buffer_multikey = snow3g_f8_4_buffer_multikey_aarch64_sve256; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_aarch64_sve256; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_aarch64_sve256; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_aarch64_sve256; - state->snow3g_init_key_sched = snow3g_init_key_sched_aarch64_sve256; - state->snow3g_key_sched_size = snow3g_key_sched_size_aarch64_sve256; - - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_aarch64; -} -void -init_mb_mgr_aarch64_sve256(IMB_MGR *state) -{ - IMB_ASSERT(state->features & IMB_FEATURE_SVE256); - init_mb_mgr_aarch64_sve256_internal(state, 1); -} -#include "mb_mgr_code.h" +#include "mb_mgr_aarch64.h" diff --git a/lib/include/arch_aarch64.h b/lib/include/arch_aarch64.h index cac79a48..26200e39 100644 --- a/lib/include/arch_aarch64.h +++ b/lib/include/arch_aarch64.h @@ -33,35 +33,102 @@ #include "ipsec-mb.h" #include "ipsec_ooo_mgr.h" -/* moved from MB MGR */ +#ifndef USED_ARCH -IMB_JOB *submit_job_zuc_eea3_aarch64(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_aarch64(MB_MGR_ZUC_OOO *state); +#define USED_ARCH IMB_ARCH_AARCH64 -IMB_JOB *submit_job_zuc256_eea3_aarch64(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_aarch64(MB_MGR_ZUC_OOO *state); +#define NUM_ZUC_LANES 4 +#define NUM_SNOW3G_LANES 4 -IMB_JOB *submit_job_zuc_eia3_aarch64(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_aarch64(MB_MGR_ZUC_OOO *state); +/* MB MGR */ +#define INIT_MB_MGR init_mb_mgr_aarch64 +#define INIT_MB_MGR_INTERNAL init_mb_mgr_internal_aarch64 +#define SUBMIT_JOB submit_job_aarch64 +#define FLUSH_JOB flush_job_aarch64 +#define QUEUE_SIZE queue_size_aarch64 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64 +#define GET_NEXT_JOB get_next_job_aarch64 +#define GET_COMPLETED_JOB get_completed_job_aarch64 +#define GET_NEXT_BURST get_next_burst_aarch64 +#define SUBMIT_BURST submit_burst_aarch64 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64 +#define FLUSH_BURST flush_burst_aarch64 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64 +#define SUBMIT_HASH_BURST submit_hash_burst_aarch64 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64 +#define SET_SUITE_ID_FN set_suite_id_aarch64 -IMB_JOB *submit_job_zuc256_eia3_aarch64(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_aarch64(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); -IMB_JOB *submit_job_snow3g_uea2_aarch64(IMB_MGR *state, IMB_JOB *job); +/* ZUC/ZUC256 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64 +#define ZUC_EEA3_1_BUFFER zuc_eea3_1_buffer_aarch64 +#define ZUC_EEA3_4_BUFFER zuc_eea3_4_buffer_aarch64 +#define ZUC_EEA3_N_BUFFER zuc_eea3_n_buffer_aarch64 +#define ZUC256_EEA_1_BUFFER zuc256_eea3_1_buffer_aarch64 +#define ZUC256_EEA_N_BUFFER zuc256_eea3_n_buffer_aarch64 +#define ZUC_EIA_1_BUFFER zuc_eia3_1_buffer_aarch64 +#define ZUC_EIA_N_BUFFER zuc_eia3_n_buffer_aarch64 +#define ZUC256_EIA3_1_BUFFER zuc256_eia3_1_buffer_aarch64 +#define ZUC256_EIA3_N_BUFFER zuc256_eia3_n_buffer_aarch64 -IMB_JOB *flush_job_snow3g_uea2_aarch64(IMB_MGR *state); +/* SNOW3G */ +#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64 +#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64 +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64 +#define SNOW3G_F8_1_BUFFER_BIT snow3g_f8_1_buffer_bit_aarch64 +#define SNOW3G_F8_1_BUFFER snow3g_f8_1_buffer_aarch64 +#define SNOW3G_F8_2_BUFFER snow3g_f8_2_buffer_aarch64 +#define SNOW3G_F8_4_BUFFER snow3g_f8_4_buffer_aarch64 +#define SNOW3G_F8_8_BUFFER snow3g_f8_8_buffer_aarch64 +#define SNOW3G_F8_N_BUFFER snow3g_f8_n_buffer_aarch64 +#define SNOW3G_F9_1_BUFFER snow3g_f9_1_buffer_aarch64 +#define SNOW3G_INIT_KEY_SCHED snow3g_init_key_sched_aarch64 +#define SNOW3G_KEY_SCHED_SIZE snow3g_key_sched_size_aarch64 +#define SNOW3G_F8_4_BUFFER_MULTIKEY snow3g_f8_4_buffer_multikey_aarch64 +#define SNOW3G_F8_8_BUFFER_MULTIKEY snow3g_f8_8_buffer_multikey_aarch64 +#define SNOW3G_F8_N_BUFFER_MULTIKEY snow3g_f8_n_buffer_multikey_aarch64 -IMB_JOB *submit_job_snow3g_uia2_aarch64(MB_MGR_SNOW3G_OOO *state,IMB_JOB *job); +#endif /* USED_ARCH */ -IMB_JOB *flush_job_snow3g_uia2_aarch64(MB_MGR_SNOW3G_OOO *state); +/* ZUC */ +IMB_JOB *SUBMIT_JOB_ZUC_EEA3(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *FLUSH_JOB_ZUC_EEA3(MB_MGR_ZUC_OOO *state); -IMB_DLL_EXPORT void set_suite_id_aarch64(IMB_MGR *state, IMB_JOB *job); +IMB_JOB *SUBMIT_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *FLUSH_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state); +IMB_JOB *SUBMIT_JOB_ZUC_EIA3(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *FLUSH_JOB_ZUC_EIA3(MB_MGR_ZUC_OOO *state); + +IMB_JOB *SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +/* SNOW3G */ +IMB_JOB *SUBMIT_JOB_SNOW3G_UEA2(IMB_MGR *state, IMB_JOB *job); + +IMB_JOB *FLUSH_JOB_SNOW3G_UEA2(IMB_MGR *state); + +IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state,IMB_JOB *job); + +IMB_JOB *FLUSH_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state); + +IMB_DLL_EXPORT void SET_SUITE_ID_FN(IMB_MGR *state, IMB_JOB *job); + +/* CRC */ uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); #endif /* ARCH_AARCH64_H */ diff --git a/lib/include/arch_aarch64_noaesni.h b/lib/include/arch_aarch64_noaesni.h index f995ba3a..30f561c8 100644 --- a/lib/include/arch_aarch64_noaesni.h +++ b/lib/include/arch_aarch64_noaesni.h @@ -30,38 +30,67 @@ #ifndef ARCH_AARCH64_NOAESNI_H #define ARCH_AARCH64_NOAESNI_H -#include "ipsec-mb.h" -#include "ipsec_ooo_mgr.h" +#define USED_ARCH IMB_ARCH_NOAESNI -/* moved from MB MGR */ +#define NUM_ZUC_LANES 4 +#define NUM_SNOW3G_LANES 4 -IMB_JOB *submit_job_zuc_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state); +/* MB MGR */ +#define INIT_MB_MGR init_mb_mgr_aarch64_no_aesni +#define INIT_MB_MGR_INTERNAL init_mb_mgr_internal_aarch64_no_aesni +#define SUBMIT_JOB submit_job_aarch64_no_aesni +#define FLUSH_JOB flush_job_aarch64_no_aesni +#define QUEUE_SIZE queue_size_aarch64_no_aesni +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_no_aesni +#define GET_NEXT_JOB get_next_job_aarch64_no_aesni +#define GET_COMPLETED_JOB get_completed_job_aarch64_no_aesni +#define GET_NEXT_BURST get_next_burst_aarch64_no_aesni +#define SUBMIT_BURST submit_burst_aarch64_no_aesni +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64_no_aesni +#define FLUSH_BURST flush_burst_aarch64_no_aesni +#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64_no_aesni +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64_no_aesni +#define SUBMIT_HASH_BURST submit_hash_burst_aarch64_no_aesni +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_no_aesni +#define SET_SUITE_ID_FN set_suite_id_aarch64_no_aesni -IMB_JOB *submit_job_zuc256_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state); +/* ZUC/ZUC256 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_no_aesni +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_no_aesni +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64_no_aesni +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64_no_aesni +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64_no_aesni +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64_no_aesni +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64_no_aesni +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64_no_aesni +#define ZUC_EEA3_1_BUFFER zuc_eea3_1_buffer_aarch64_no_aesni +#define ZUC_EEA3_4_BUFFER zuc_eea3_4_buffer_aarch64_no_aesni +#define ZUC_EEA3_N_BUFFER zuc_eea3_n_buffer_aarch64_no_aesni +#define ZUC256_EEA_1_BUFFER zuc256_eea3_1_buffer_aarch64_no_aesni +#define ZUC256_EEA_N_BUFFER zuc256_eea3_n_buffer_aarch64_no_aesni +#define ZUC_EIA_1_BUFFER zuc_eia3_1_buffer_aarch64_no_aesni +#define ZUC_EIA_N_BUFFER zuc_eia3_n_buffer_aarch64_no_aesni +#define ZUC256_EIA3_1_BUFFER zuc256_eia3_1_buffer_aarch64_no_aesni +#define ZUC256_EIA3_N_BUFFER zuc256_eia3_n_buffer_aarch64_no_aesni -IMB_JOB *submit_job_zuc_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state); +/* SNOW3G */ +#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_no_aesni +#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_no_aesni +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_no_aesni +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_no_aesni +#define SNOW3G_F8_1_BUFFER_BIT snow3g_f8_1_buffer_bit_aarch64_no_aesni +#define SNOW3G_F8_1_BUFFER snow3g_f8_1_buffer_aarch64_no_aesni +#define SNOW3G_F8_2_BUFFER snow3g_f8_2_buffer_aarch64_no_aesni +#define SNOW3G_F8_4_BUFFER snow3g_f8_4_buffer_aarch64_no_aesni +#define SNOW3G_F8_8_BUFFER snow3g_f8_8_buffer_aarch64_no_aesni +#define SNOW3G_F8_N_BUFFER snow3g_f8_n_buffer_aarch64_no_aesni +#define SNOW3G_F9_1_BUFFER snow3g_f9_1_buffer_aarch64_no_aesni +#define SNOW3G_INIT_KEY_SCHED snow3g_init_key_sched_aarch64_no_aesni +#define SNOW3G_KEY_SCHED_SIZE snow3g_key_sched_size_aarch64_no_aesni +#define SNOW3G_F8_4_BUFFER_MULTIKEY snow3g_f8_4_buffer_multikey_aarch64_no_aesni +#define SNOW3G_F8_8_BUFFER_MULTIKEY snow3g_f8_8_buffer_multikey_aarch64_no_aesni +#define SNOW3G_F8_N_BUFFER_MULTIKEY snow3g_f8_n_buffer_multikey_aarch64_no_aesni -IMB_JOB *submit_job_zuc256_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); -IMB_JOB *submit_job_snow3g_uea2_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uea2_aarch64_no_aesni(IMB_MGR *state); - -IMB_JOB *submit_job_snow3g_uia2_aarch64_no_aesni(MB_MGR_SNOW3G_OOO *state,IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uia2_aarch64_no_aesni(MB_MGR_SNOW3G_OOO *state); - -IMB_DLL_EXPORT void set_suite_id_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *job); - -uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); +#include "arch_aarch64.h" #endif /* ARCH_AARCH64_NOAESNI_H */ diff --git a/lib/include/arch_aarch64_sve256.h b/lib/include/arch_aarch64_sve256.h index 69c7714c..88b4454a 100644 --- a/lib/include/arch_aarch64_sve256.h +++ b/lib/include/arch_aarch64_sve256.h @@ -30,38 +30,67 @@ #ifndef ARCH_AARCH64_SVE256_H #define ARCH_AARCH64_SVE256_H -#include "ipsec-mb.h" -#include "ipsec_ooo_mgr.h" +#define USED_ARCH IMB_ARCH_SVE256 -/* moved from MB MGR */ +#define NUM_ZUC_LANES 4 +#define NUM_SNOW3G_LANES 8 -IMB_JOB *submit_job_zuc_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state); +/* MB MGR */ +#define INIT_MB_MGR init_mb_mgr_aarch64_sve256 +#define INIT_MB_MGR_INTERNAL init_mb_mgr_internal_aarch64_sve256 +#define SUBMIT_JOB submit_job_aarch64_sve256 +#define FLUSH_JOB flush_job_aarch64_sve256 +#define QUEUE_SIZE queue_size_aarch64_sve256 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_sve256 +#define GET_NEXT_JOB get_next_job_aarch64_sve256 +#define GET_COMPLETED_JOB get_completed_job_aarch64_sve256 +#define GET_NEXT_BURST get_next_burst_aarch64_sve256 +#define SUBMIT_BURST submit_burst_aarch64_sve256 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64_sve256 +#define FLUSH_BURST flush_burst_aarch64_sve256 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64_sve256 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64_sve256 +#define SUBMIT_HASH_BURST submit_hash_burst_aarch64_sve256 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_sve256 +#define SET_SUITE_ID_FN set_suite_id_aarch64_sve256 -IMB_JOB *submit_job_zuc256_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state); +/* ZUC/ZUC256 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64 +#define ZUC_EEA3_1_BUFFER zuc_eea3_1_buffer_aarch64 +#define ZUC_EEA3_4_BUFFER zuc_eea3_4_buffer_aarch64 +#define ZUC_EEA3_N_BUFFER zuc_eea3_n_buffer_aarch64 +#define ZUC256_EEA_1_BUFFER zuc256_eea3_1_buffer_aarch64 +#define ZUC256_EEA_N_BUFFER zuc256_eea3_n_buffer_aarch64 +#define ZUC_EIA_1_BUFFER zuc_eia3_1_buffer_aarch64 +#define ZUC_EIA_N_BUFFER zuc_eia3_n_buffer_aarch64 +#define ZUC256_EIA3_1_BUFFER zuc256_eia3_1_buffer_aarch64 +#define ZUC256_EIA3_N_BUFFER zuc256_eia3_n_buffer_aarch64 -IMB_JOB *submit_job_zuc_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state); +/* SNOW3G */ +#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_sve256 +#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_sve256 +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_sve256 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_sve256 +#define SNOW3G_F8_1_BUFFER_BIT snow3g_f8_1_buffer_bit_aarch64_sve256 +#define SNOW3G_F8_1_BUFFER snow3g_f8_1_buffer_aarch64_sve256 +#define SNOW3G_F8_2_BUFFER snow3g_f8_2_buffer_aarch64_sve256 +#define SNOW3G_F8_4_BUFFER snow3g_f8_4_buffer_aarch64_sve256 +#define SNOW3G_F8_8_BUFFER snow3g_f8_8_buffer_aarch64_sve256 +#define SNOW3G_F8_N_BUFFER snow3g_f8_n_buffer_aarch64_sve256 +#define SNOW3G_F9_1_BUFFER snow3g_f9_1_buffer_aarch64_sve256 +#define SNOW3G_INIT_KEY_SCHED snow3g_init_key_sched_aarch64_sve256 +#define SNOW3G_KEY_SCHED_SIZE snow3g_key_sched_size_aarch64_sve256 +#define SNOW3G_F8_4_BUFFER_MULTIKEY snow3g_f8_4_buffer_multikey_aarch64_sve256 +#define SNOW3G_F8_8_BUFFER_MULTIKEY snow3g_f8_8_buffer_multikey_aarch64_sve256 +#define SNOW3G_F8_N_BUFFER_MULTIKEY snow3g_f8_n_buffer_multikey_aarch64_sve256 -IMB_JOB *submit_job_zuc256_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); -IMB_JOB *submit_job_snow3g_uea2_aarch64_sve256(IMB_MGR *state, IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uea2_aarch64_sve256(IMB_MGR *state); - -IMB_JOB *submit_job_snow3g_uia2_aarch64_sve256(MB_MGR_SNOW3G_OOO *state,IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uia2_aarch64_sve256(MB_MGR_SNOW3G_OOO *state); - -IMB_DLL_EXPORT void set_suite_id_aarch64_sve256(IMB_MGR *state, IMB_JOB *job); - -uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); +#include "arch_aarch64.h" #endif /* ARCH_AARCH64_SVE256_H */ diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 4b67cadd..ac69e064 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -965,7 +965,7 @@ get_completed_job_avx512_t2(IMB_MGR *state); #ifdef __aarch64__ /* noaesni functions */ IMB_DLL_LOCAL void -init_mb_mgr_aarch64_no_aesni_internal(IMB_MGR *state, const int reset_mgrs); +init_mb_mgr_internal_aarch64_no_aesni(IMB_MGR *state, const int reset_mgrs); IMB_DLL_EXPORT uint32_t get_next_burst_aarch64_no_aesni(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -1004,7 +1004,7 @@ IMB_DLL_EXPORT IMB_JOB *get_completed_job_aarch64_no_aesni(IMB_MGR *state); /* aarch64 functions */ IMB_DLL_LOCAL void -init_mb_mgr_aarch64_internal(IMB_MGR *state, const int reset_mgrs); +init_mb_mgr_internal_aarch64(IMB_MGR *state, const int reset_mgrs); IMB_DLL_EXPORT uint32_t get_next_burst_aarch64(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -1043,7 +1043,7 @@ IMB_DLL_EXPORT IMB_JOB *get_completed_job_aarch64(IMB_MGR *state); /* sve256 functions */ IMB_DLL_LOCAL void -init_mb_mgr_aarch64_sve256_internal(IMB_MGR *state, const int reset_mgrs); +init_mb_mgr_internal_aarch64_sve256(IMB_MGR *state, const int reset_mgrs); IMB_DLL_EXPORT uint32_t get_next_burst_aarch64_sve256(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 78991f7c..15daae83 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -231,10 +231,10 @@ imb_set_pointers_mb_mgr(void *mem_ptr, const uint64_t flags, const unsigned rese break; #else case IMB_ARCH_NOAESNI: - init_mb_mgr_aarch64_no_aesni_internal(ptr, 0); + init_mb_mgr_internal_aarch64_no_aesni(ptr, 0); break; case IMB_ARCH_AARCH64: - init_mb_mgr_aarch64_internal(ptr, 0); + init_mb_mgr_internal_aarch64(ptr, 0); break; #endif default: -- GitLab From 10956bef693dcc8cd2311b9aa2cbb63c53cd2570 Mon Sep 17 00:00:00 2001 From: "fisher.yu" Date: Wed, 8 Jan 2025 11:45:06 +0000 Subject: [PATCH 2/3] lib: [sha1] add sha1 implementation for aarch64 platform - add 4-buffer SHA1 implementation - support job/burst API for SHA1 - uncomment corresponding kat/xvalid-app tests Signed-off-by: Islam Ragimov --- lib/Makefile | 2 + lib/aarch64/mb_mgr_aarch64.h | 8 + lib/aarch64/sha1_mb_neon_x4.S | 204 +++++++++++++++++++++ lib/aarch64/sha1_neon_common.inc | 274 +++++++++++++++++++++++++++++ lib/aarch64/sha_job_neon.c | 61 +++++++ lib/include/arch_aarch64.h | 11 ++ lib/include/arch_aarch64_noaesni.h | 4 + lib/include/arch_aarch64_sve256.h | 4 + lib/include/ipsec_ooo_mgr.h | 1 + lib/include/mb_mgr_burst.h | 9 +- lib/include/mb_mgr_job_api.h | 33 +++- lib/include/mb_mgr_job_check.h | 4 + lib/include/ooo_mgr_reset.h | 2 +- lib/x86_64/ooo_mgr_reset.c | 13 +- perf/ipsec_perf.c | 2 + test/kat-app/CMakeLists.txt | 2 + test/kat-app/Makefile | 3 +- test/kat-app/main.c | 2 + test/kat-app/sha_test.c | 11 +- test/xvalid-app/ipsec_xvalid.c | 8 + 20 files changed, 645 insertions(+), 13 deletions(-) create mode 100644 lib/aarch64/sha1_mb_neon_x4.S create mode 100644 lib/aarch64/sha1_neon_common.inc create mode 100644 lib/aarch64/sha_job_neon.c diff --git a/lib/Makefile b/lib/Makefile index f6fd7e7e..eb72ed36 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -367,6 +367,7 @@ c_lib_objs := \ zuc_aarch64_no_aesni_top.o \ zuc_aarch64_top.o \ crc32_aarch64.o \ + sha_job_neon.o \ mb_mgr_zuc_submit_flush_aarch64.o \ mb_mgr_zuc_submit_flush_aarch64_no_aesni.o \ mb_mgr_snow3g_submit_flush_aarch64.o \ @@ -382,6 +383,7 @@ asm_generic_lib_objs := \ lookup_16x8bit_neon.o \ snow3g_impl_aarch64_neon.o \ snow3g_impl_aarch64_sve256.o \ + sha1_mb_neon_x4.o \ zuc_simd.o \ zuc_simd_no_aesni.o else diff --git a/lib/aarch64/mb_mgr_aarch64.h b/lib/aarch64/mb_mgr_aarch64.h index 0e7ccda7..9830f654 100644 --- a/lib/aarch64/mb_mgr_aarch64.h +++ b/lib/aarch64/mb_mgr_aarch64.h @@ -47,6 +47,9 @@ #include "include/ooo_mgr_reset.h" +IMB_JOB *(*SUBMIT_JOB_SHA1)(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *(*FLUSH_JOB_SHA1)(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); + static void reset_ooo_mgrs(IMB_MGR *state) { /* Init ZUC out-of-order fields */ @@ -55,6 +58,9 @@ static void reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, NUM_ZUC_LANES); ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, NUM_ZUC_LANES); + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, NEON_NUM_SHA1_LANES); + /* Init SNOW3G-UEA out-of-order fields */ ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, NUM_SNOW3G_LANES); @@ -112,6 +118,8 @@ INIT_MB_MGR_INTERNAL(IMB_MGR *state, const int reset_mgrs) state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; state->set_suite_id = SET_SUITE_ID_FN; + SUBMIT_JOB_SHA1 = submit_job_sha1_neon; + FLUSH_JOB_SHA1 = flush_job_sha1_neon; state->eea3_1_buffer = ZUC_EEA3_1_BUFFER; state->eea3_4_buffer = ZUC_EEA3_4_BUFFER; diff --git a/lib/aarch64/sha1_mb_neon_x4.S b/lib/aarch64/sha1_mb_neon_x4.S new file mode 100644 index 00000000..edf47482 --- /dev/null +++ b/lib/aarch64/sha1_mb_neon_x4.S @@ -0,0 +1,204 @@ +/********************************************************************** + SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its + affiliates + SPDX-License-Identifier: BSD-3-Clause + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .arch armv8-a + +#include "sha1_neon_common.inc" + +.macro internal_load windex + // load 64-bytes from each address to maximize usage of cache line + .if \windex == 0 + mov tmp,dataptr + ld1 {WORD0.4s},[data0],16 + ld1 {WORD4.4s},[data0],16 + ld1 {WORD8.4s},[data0],16 + ld1 {WORD12.4s},[data0],16 + + ld1 {WORD1.4s},[data1],16 + ld1 {WORD5.4s},[data1],16 + ld1 {WORD9.4s},[data1],16 + ld1 {WORD13.4s},[data1],16 + + ld1 {WORD2.4s},[data2],16 + ld1 {WORD6.4s},[data2],16 + ld1 {WORD10.4s},[data2],16 + ld1 {WORD14.4s},[data2],16 + + ld1 {WORD3.4s},[data3],16 + ld1 {WORD7.4s},[data3],16 + ld1 {WORD11.4s},[data3],16 + ld1 {WORD15.4s},[data3],16 + + st4 {WORD0.s,WORD1.s,WORD2.s,WORD3.s}[0],[tmp],16 + st4 {WORD0.s,WORD1.s,WORD2.s,WORD3.s}[1],[tmp],16 + st4 {WORD0.s,WORD1.s,WORD2.s,WORD3.s}[2],[tmp],16 + st4 {WORD0.s,WORD1.s,WORD2.s,WORD3.s}[3],[tmp],16 + .endif + + .if \windex == 4 + mov tmp,dataptr + st4 {WORD4.s,WORD5.s,WORD6.s,WORD7.s}[0],[tmp],16 + st4 {WORD4.s,WORD5.s,WORD6.s,WORD7.s}[1],[tmp],16 + st4 {WORD4.s,WORD5.s,WORD6.s,WORD7.s}[2],[tmp],16 + st4 {WORD4.s,WORD5.s,WORD6.s,WORD7.s}[3],[tmp],16 + .endif + + .if \windex == 8 + mov tmp,dataptr + st4 {WORD8.s,WORD9.s,WORD10.s,WORD11.s}[0],[tmp],16 + st4 {WORD8.s,WORD9.s,WORD10.s,WORD11.s}[1],[tmp],16 + st4 {WORD8.s,WORD9.s,WORD10.s,WORD11.s}[2],[tmp],16 + st4 {WORD8.s,WORD9.s,WORD10.s,WORD11.s}[3],[tmp],16 + .endif + + .if \windex == 12 + mov tmp,dataptr + st4 {WORD12.s,WORD13.s,WORD14.s,WORD15.s}[0],[tmp],16 + st4 {WORD12.s,WORD13.s,WORD14.s,WORD15.s}[1],[tmp],16 + st4 {WORD12.s,WORD13.s,WORD14.s,WORD15.s}[2],[tmp],16 + st4 {WORD12.s,WORD13.s,WORD14.s,WORD15.s}[3],[tmp],16 + .endif +.endm + +.macro load_x4_word idx:req + internal_load \idx + ld1 {WORD\idx\().16b},[dataptr],16 +.endm + +/* + * void sha1_mb_neon_x4(SHA1_ARGS *args, int blocks) + */ + args .req x0 + num_blocks .req w1 + digest0 .req x13 + digest1 .req x14 + digest2 .req x2 + digest3 .req x3 + tmp .req x5 + data0 .req x6 + data1 .req x7 + data2 .req x8 + data3 .req x9 + databuf .req x10 + dataptr .req x11 + savedsp .req x12 + + .global sha1_mb_neon_x4 + .type sha1_mb_neon_x4, %function +sha1_mb_neon_x4: + cmp num_blocks, #0 + beq .return + sha1_asimd_save_stack + mov savedsp,sp + sub databuf,sp,256 + mov tmp,63 + bic databuf,databuf,tmp + mov sp,databuf + + add digest0, args, #0 + add digest1, args, #20 + add digest2, args, #40 + add digest3, args, #60 + ldp data0, data1, [args, #320] + ldp data2, data3, [args, #336] + + mov tmp, digest0 + ld4 {VA.s,VB.s,VC.s,VD.s}[0],[tmp],#16 + ld1 {VE.s}[0],[tmp] + + mov tmp, digest1 + ld4 {VA.s,VB.s,VC.s,VD.s}[1],[tmp],#16 + ld1 {VE.s}[1],[tmp] + + mov tmp, digest2 + ld4 {VA.s,VB.s,VC.s,VD.s}[2],[tmp],#16 + ld1 {VE.s}[2],[tmp] + + mov tmp, digest3 + ld4 {VA.s,VB.s,VC.s,VD.s}[3],[tmp],#16 + ld1 {VE.s}[3],[tmp] + +.block_loop: + mov dataptr,databuf + sha1_single + subs num_blocks, num_blocks, 1 + bne .block_loop + + stp data0, data1, [args, #320] + stp data2, data3, [args, #336] + + mov tmp, digest0 + st4 {VA.s,VB.s,VC.s,VD.s}[0],[tmp],#16 + st1 {VE.s}[0],[tmp] + + mov tmp, digest1 + st4 {VA.s,VB.s,VC.s,VD.s}[1],[tmp],#16 + st1 {VE.s}[1],[tmp] + + mov tmp, digest2 + st4 {VA.s,VB.s,VC.s,VD.s}[2],[tmp],#16 + st1 {VE.s}[2],[tmp] + + mov tmp, digest3 + st4 {VA.s,VB.s,VC.s,VD.s}[3],[tmp],#16 + st1 {VE.s}[3],[tmp] + + mov sp,savedsp + sha1_asimd_restore_stack +.return: + ret + + .size sha1_mb_neon_x4, .-sha1_mb_neon_x4 + +.align 4 +KEY_0: + .word 0x5a827999 + .word 0x5a827999 + .word 0x5a827999 + .word 0x5a827999 +KEY_1: + .word 0x6ed9eba1 + .word 0x6ed9eba1 + .word 0x6ed9eba1 + .word 0x6ed9eba1 +KEY_2: + .word 0x8f1bbcdc + .word 0x8f1bbcdc + .word 0x8f1bbcdc + .word 0x8f1bbcdc +KEY_3: + .word 0xca62c1d6 + .word 0xca62c1d6 + .word 0xca62c1d6 + .word 0xca62c1d6 diff --git a/lib/aarch64/sha1_neon_common.inc b/lib/aarch64/sha1_neon_common.inc new file mode 100644 index 00000000..4c46c222 --- /dev/null +++ b/lib/aarch64/sha1_neon_common.inc @@ -0,0 +1,274 @@ +/********************************************************************** + SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its + affiliates + SPDX-License-Identifier: BSD-3-Clause + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .arch armv8-a + +// macro F = (D ^ (B & (C ^ D))) +.macro FUNC_F0 + eor VF.16b, VC.16b, VD.16b + and VF.16b, VB.16b, VF.16b + eor VF.16b, VD.16b, VF.16b +.endm + +// F = (B ^ C ^ D) +.macro FUNC_F1 + eor VF.16b, VB.16b, VC.16b + eor VF.16b, VF.16b, VD.16b +.endm + +// F = ((B & C) | (B & D) | (C & D)) +.macro FUNC_F2 + and vT0.16b, VB.16b, VC.16b + and vT1.16b, VB.16b, VD.16b + and vT2.16b, VC.16b, VD.16b + orr VF.16b, vT0.16b, vT1.16b + orr VF.16b, VF.16b, vT2.16b +.endm + +// F = (B ^ C ^ D) +.macro FUNC_F3 + FUNC_F1 +.endm + +.altmacro +.macro load_next_word windex + .if \windex < 16 + load_x4_word \windex + .endif +.endm + +// FUNC_F0 is merged into STEP_00_15 for efficiency +.macro SHA1_STEP_00_15_F0 windex:req + rev32 WORD\windex\().16b,WORD\windex\().16b + next_word=\windex+1 + load_next_word %next_word + // e = (a leftrotate 5) + f + e + k + w[i] + ushr VT.4s, VA.4s, 32 - 5 + add VE.4s, VE.4s, VK.4s + sli VT.4s, VA.4s, 5 + eor VF.16b, VC.16b, VD.16b + add VE.4s, VE.4s, WORD\windex\().4s + and VF.16b, VB.16b, VF.16b + add VE.4s, VE.4s, VT.4s + eor VF.16b, VD.16b, VF.16b + ushr VT.4s, VB.4s, 32 - 30 + add VE.4s, VE.4s, VF.4s + sli VT.4s, VB.4s, 30 +.endm + +.macro SHA1_STEP_16_79 windex:req,func_f:req,reg_3:req,reg_8:req,reg_14:req,reg_16:req + eor vT0.16b,\reg_3\().16b,\reg_8\().16b + eor VT.16b,\reg_14\().16b,\reg_16\().16b + eor vT0.16b,vT0.16b,VT.16b + // e = (a leftrotate 5) + f + e + k + w[i] + ushr VT.4s, vT0.4s, 32 - 1 + add VE.4s, VE.4s, VK.4s + ushr vT1.4s, VA.4s, 32 - 5 + sli VT.4s, vT0.4s, 1 + add VE.4s, VE.4s, VT.4s + sli vT1.4s, VA.4s, 5 + mov \reg_16\().16b,VT.16b + add VE.4s, VE.4s, vT1.4s + ushr VT.4s, VB.4s, 32 - 30 + \func_f + add VE.4s, VE.4s, VF.4s + sli VT.4s, VB.4s, 30 +.endm + + VA .req v0 + VB .req v1 + VC .req v2 + VD .req v3 + VE .req v4 + VT .req v5 + VF .req v6 + VK .req v7 + WORD0 .req v8 + WORD1 .req v9 + WORD2 .req v10 + WORD3 .req v11 + WORD4 .req v12 + WORD5 .req v13 + WORD6 .req v14 + WORD7 .req v15 + WORD8 .req v16 + WORD9 .req v17 + WORD10 .req v18 + WORD11 .req v19 + WORD12 .req v20 + WORD13 .req v21 + WORD14 .req v22 + WORD15 .req v23 + vT0 .req v24 + vT1 .req v25 + vT2 .req v26 + vAA .req v27 + vBB .req v28 + vCC .req v29 + vDD .req v30 + vEE .req v31 + TT .req v0 + sha1key_adr .req x15 + +.macro SWAP_STATES + // shifted VB is held in VT after each step + .unreq TT + TT .req VE + .unreq VE + VE .req VD + .unreq VD + VD .req VC + .unreq VC + VC .req VT + .unreq VT + VT .req VB + .unreq VB + VB .req VA + .unreq VA + VA .req TT +.endm + +.altmacro +.macro SHA1_STEP_16_79_WRAPPER windex:req,func_f:req,idx3:req,idx8:req,idx14:req,idx16:req + SHA1_STEP_16_79 \windex,\func_f,WORD\idx3\(),WORD\idx8\(),WORD\idx14\(),WORD\idx16\() +.endm + +.macro exec_step windex:req + .if \windex <= 15 + SHA1_STEP_00_15_F0 windex + .else + idx14=((\windex - 14) & 15) + idx8=((\windex - 8) & 15) + idx3=((\windex - 3) & 15) + idx16=(\windex & 15) + .if \windex <= 19 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F0,%idx3,%idx8,%idx14,%idx16 + .endif + .if \windex >= 20 && \windex <= 39 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F1,%idx3,%idx8,%idx14,%idx16 + .endif + .if \windex >= 40 && \windex <= 59 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F2,%idx3,%idx8,%idx14,%idx16 + .endif + .if \windex >= 60 && \windex <= 79 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F3,%idx3,%idx8,%idx14,%idx16 + .endif + .endif + + SWAP_STATES + + .if \windex == 79 + // after 80 steps, the registers ABCDET has shifted from + // its orignal order of 012345 to 341520 + // have to swap back for both compile- and run-time correctness + mov v0.16b,v3.16b + .unreq VA + VA .req v0 + + mov vT0.16b,v2.16b + mov v2.16b,v1.16b + mov v1.16b,v4.16b + .unreq VB + VB .req v1 + .unreq VC + VC .req v2 + + mov v3.16b,v5.16b + .unreq VD + VD .req v3 + + mov v4.16b,vT0.16b + .unreq VE + VE .req v4 + + .unreq VT + VT .req v5 + .endif +.endm + +.macro exec_steps idx:req,more:vararg + exec_step \idx + .ifnb \more + exec_steps \more + .endif +.endm + +.macro sha1_single + load_x4_word 0 + + mov vAA.16B, VA.16B + mov vBB.16B, VB.16B + mov vCC.16B, VC.16B + mov vDD.16B, VD.16B + mov vEE.16B, VE.16B + + adr sha1key_adr, KEY_0 + ld1 {VK.4s}, [sha1key_adr] + exec_steps 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19 + + // 20 ~ 39 + adr sha1key_adr, KEY_1 + ld1 {VK.4s}, [sha1key_adr] + exec_steps 20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 + + // 40 ~ 59 + adr sha1key_adr, KEY_2 + ld1 {VK.4s}, [sha1key_adr] + exec_steps 40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59 + + // 60 ~ 79 + adr sha1key_adr, KEY_3 + ld1 {VK.4s}, [sha1key_adr] + exec_steps 60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79 + + add VA.4s, vAA.4s, VA.4s + add VB.4s, vBB.4s, VB.4s + add VC.4s, vCC.4s, VC.4s + add VD.4s, vDD.4s, VD.4s + add VE.4s, vEE.4s, VE.4s +.endm + +.macro sha1_asimd_save_stack + stp d8,d9,[sp, -64]! + stp d10,d11,[sp, 16] + stp d12,d13,[sp, 32] + stp d14,d15,[sp, 48] +.endm + +.macro sha1_asimd_restore_stack + ldp d10,d11,[sp, 16] + ldp d12,d13,[sp, 32] + ldp d14,d15,[sp, 48] + ldp d8,d9,[sp],64 +.endm diff --git a/lib/aarch64/sha_job_neon.c b/lib/aarch64/sha_job_neon.c new file mode 100644 index 00000000..b8f74741 --- /dev/null +++ b/lib/aarch64/sha_job_neon.c @@ -0,0 +1,61 @@ +/********************************************************************** + SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its + affiliates + SPDX-License-Identifier: BSD-3-Clause + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ +#define SHA1_MAX_JOBS NEON_NUM_SHA1_LANES +#define FLUSH_JOB_SHA1_IMPL flush_job_sha1_neon +#define SUBMIT_JOB_SHA1_IMPL submit_job_sha1_neon +#define SHA1_MB_IMPL sha1_mb_neon_x4 + +#include "include/sha_mb_mgr.h" +#include "include/arch_aarch64.h" + +/* ========================================================================== */ +/* + * SHA1 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *SUBMIT_JOB_SHA1_IMPL(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, SHA1_MAX_JOBS, 1, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + SHA1_MB_IMPL, 1); +} + +IMB_DLL_LOCAL +IMB_JOB *FLUSH_JOB_SHA1_IMPL(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, SHA1_MAX_JOBS, 0, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + SHA1_MB_IMPL, 1); +} diff --git a/lib/include/arch_aarch64.h b/lib/include/arch_aarch64.h index 26200e39..08411344 100644 --- a/lib/include/arch_aarch64.h +++ b/lib/include/arch_aarch64.h @@ -59,6 +59,10 @@ #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64 #define SET_SUITE_ID_FN set_suite_id_aarch64 +/* SHA1 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_aarch64 +#define FLUSH_JOB_SHA1 flush_job_sha1_aarch64 + /* ZUC/ZUC256 */ #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 @@ -131,4 +135,11 @@ IMB_DLL_EXPORT void SET_SUITE_ID_FN(IMB_MGR *state, IMB_JOB *job); /* CRC */ uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); +/* SHA */ +void sha1_mb_neon_x4(SHA1_ARGS *args, uint32_t size_in_blocks); + +IMB_JOB *submit_job_sha1_neon(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_neon(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); #endif /* ARCH_AARCH64_H */ diff --git a/lib/include/arch_aarch64_noaesni.h b/lib/include/arch_aarch64_noaesni.h index 30f561c8..aef53c9e 100644 --- a/lib/include/arch_aarch64_noaesni.h +++ b/lib/include/arch_aarch64_noaesni.h @@ -54,6 +54,10 @@ #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_no_aesni #define SET_SUITE_ID_FN set_suite_id_aarch64_no_aesni +/* SHA1 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_aarch64_no_aesni +#define FLUSH_JOB_SHA1 flush_job_sha1_aarch64_no_aesni + /* ZUC/ZUC256 */ #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_no_aesni #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_no_aesni diff --git a/lib/include/arch_aarch64_sve256.h b/lib/include/arch_aarch64_sve256.h index 88b4454a..f5fafd1f 100644 --- a/lib/include/arch_aarch64_sve256.h +++ b/lib/include/arch_aarch64_sve256.h @@ -54,6 +54,10 @@ #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_sve256 #define SET_SUITE_ID_FN set_suite_id_aarch64_sve256 +/* SHA1 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_aarch64_sve256 +#define FLUSH_JOB_SHA1 flush_job_sha1_aarch64_sve256 + /* ZUC/ZUC256 */ #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index ac69e064..166f6e34 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -65,6 +65,7 @@ #define SSE_NUM_SHA512_LANES AVX_NUM_SHA512_LANES #define SSE_NUM_MD5_LANES AVX_NUM_MD5_LANES +#define NEON_NUM_SHA1_LANES 4 /* * Each row is sized to hold enough lanes for AVX2, AVX1 and SSE use a subset * of each row. Thus one row is not adjacent in memory to its neighboring rows diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index 7d870920..361ec1f1 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -691,6 +691,7 @@ submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, co return completed_jobs; } +#endif /* __aarch64__ */ __forceinline uint32_t submit_burst_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const int run_check, @@ -732,7 +733,9 @@ submit_burst_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const i completed_jobs++; } } - } else if (hash_alg == IMB_AUTH_SHA_224) { + } +#ifndef __aarch64__ + else if (hash_alg == IMB_AUTH_SHA_224) { /* submit all jobs */ for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; @@ -813,10 +816,10 @@ submit_burst_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const i } } } +#endif /* __aarch64__ */ return completed_jobs; } -#endif /* __aarch64__ */ __forceinline uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -848,8 +851,10 @@ submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs case IMB_AUTH_HMAC_SHA_512: return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_512); +#endif /* __aarch64__ */ case IMB_AUTH_SHA_1: return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_1); +#ifndef __aarch64__ case IMB_AUTH_SHA_224: return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_224); case IMB_AUTH_SHA_256: diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index a776f96f..404c8c90 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -2666,7 +2666,9 @@ SUBMIT_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) #ifndef __aarch64__ MB_MGR_ZUC_OOO *zuc256_eia3_8B_ooo = state->zuc256_eia3_8B_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_16B_ooo = state->zuc256_eia3_16B_ooo; +#endif /* __aarch64__ */ MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; +#ifndef __aarch64__ MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; @@ -2716,8 +2718,10 @@ SUBMIT_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) case IMB_AUTH_AES_CMAC_256: job->msg_len_to_hash_in_bits = job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); +#endif /* __aarch64__ */ case IMB_AUTH_SHA_1: return SUBMIT_JOB_SHA1(sha_1_ooo, job); +#ifndef __aarch64__ case IMB_AUTH_SHA_224: return SUBMIT_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: @@ -2865,7 +2869,9 @@ FLUSH_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) #ifndef __aarch64__ MB_MGR_ZUC_OOO *zuc256_eia3_8B_ooo = state->zuc256_eia3_8B_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_16B_ooo = state->zuc256_eia3_16B_ooo; +#endif /* __aarch64__ */ MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; +#ifndef __aarch64__ MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; @@ -2889,8 +2895,10 @@ FLUSH_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); +#endif case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); +#ifndef __aarch64__ case IMB_AUTH_SHA_224: return FLUSH_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: @@ -3024,6 +3032,7 @@ submit_hash_aes_cmac(IMB_MGR *state, IMB_JOB *job) { return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC); } +#endif /* __aarch64__ */ static IMB_JOB * submit_hash_sha1(IMB_MGR *state, IMB_JOB *job) @@ -3031,6 +3040,7 @@ submit_hash_sha1(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SHA_1); } +#ifndef __aarch64__ static IMB_JOB * submit_hash_sha224(IMB_MGR *state, IMB_JOB *job) { @@ -3282,8 +3292,13 @@ static const submit_flush_fn_t tab_submit_hash[] = { submit_hash_aes_ccm, /* [12] AES-CMAC */ submit_hash_aes_cmac, +#else /* __aarch64__ */ + /* [9] - [12] NULL */ + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ /* [13] SHA1 */ submit_hash_sha1, +#ifndef __aarch64__ /* [14] SHA224 */ submit_hash_sha224, /* [15] SHA256 */ @@ -3297,10 +3312,9 @@ static const submit_flush_fn_t tab_submit_hash[] = { /* [19] PON CRC BIP */ submit_hash_pon_crc_bip, #else /* __aarch64__ */ - /* [9] - [19] NULL */ - NULL, NULL, NULL, NULL, + /* [14] - [19] NULL */ NULL, NULL, NULL, NULL, - NULL, NULL, NULL, + NULL, NULL, #endif /* __aarch64__ */ /* [20] ZUC EIA3 BIT */ submit_hash_zuc_eia3_bit, @@ -3386,7 +3400,6 @@ static const submit_flush_fn_t tab_submit_hash[] = { /* ========================================================================= */ /* Generate specialized hash flush functions and create a table */ /* ========================================================================= */ - #ifndef __aarch64__ static IMB_JOB * flush_hash_hmac_sha1(IMB_MGR *state, IMB_JOB *job) @@ -3461,6 +3474,7 @@ flush_hash_aes_cmac(IMB_MGR *state, IMB_JOB *job) { return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC); } +#endif /* __aarch64__ */ static IMB_JOB * flush_hash_sha1(IMB_MGR *state, IMB_JOB *job) @@ -3468,6 +3482,7 @@ flush_hash_sha1(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SHA_1); } +#ifndef __aarch64__ static IMB_JOB * flush_hash_sha224(IMB_MGR *state, IMB_JOB *job) { @@ -3719,8 +3734,13 @@ static const submit_flush_fn_t tab_flush_hash[] = { flush_hash_aes_ccm, /* [12] AES-CMAC */ flush_hash_aes_cmac, +#else /* __aarch64__ */ + /* [9] - [12] NULL */ + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ /* [13] SHA1 */ flush_hash_sha1, +#ifndef __aarch64__ /* [14] SHA224 */ flush_hash_sha224, /* [15] SHA256 */ @@ -3734,10 +3754,9 @@ static const submit_flush_fn_t tab_flush_hash[] = { /* [19] PON CRC BIP */ flush_hash_pon_crc_bip, #else /* __aarch64__ */ - /* [9] - [19] NULL */ - NULL, NULL, NULL, NULL, + /* [14] - [19] NULL */ NULL, NULL, NULL, NULL, - NULL, NULL, NULL, + NULL, NULL, #endif /* __aarch64__ */ /* [20] ZUC EIA3 BIT */ flush_hash_zuc_eia3_bit, diff --git a/lib/include/mb_mgr_job_check.h b/lib/include/mb_mgr_job_check.h index 05fc71d6..268936aa 100644 --- a/lib/include/mb_mgr_job_check.h +++ b/lib/include/mb_mgr_job_check.h @@ -1528,11 +1528,14 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, const IMB_CIPHER_MODE cipher_ return 1; } break; +#endif /* __aarch64__ */ case IMB_AUTH_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_SHA_224: case IMB_AUTH_SHA_256: case IMB_AUTH_SHA_384: case IMB_AUTH_SHA_512: +#endif /* __aarch64__ */ if (job->auth_tag_output_len_in_bytes != auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; @@ -1550,6 +1553,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, const IMB_CIPHER_MODE cipher_ return 1; } break; +#ifndef __aarch64__ case IMB_AUTH_PON_CRC_BIP: /* * Authentication tag in PON is BIP 32-bit value only diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index 416fff20..7b153398 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -74,11 +74,11 @@ IMB_DLL_LOCAL void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes); -#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes); diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 098d0504..d8fd4155 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -345,7 +345,6 @@ ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes) } } -#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) @@ -355,6 +354,7 @@ ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, offsetof(MB_MGR_SHA_1_OOO, road_block)); p_mgr->total_num_lanes = num_lanes; +#ifndef __aarch64__ if (num_lanes == 2) p_mgr->unused_lanes = 0xF10; /* SHANI */ else if (num_lanes == AVX_NUM_SHA1_LANES) @@ -363,8 +363,19 @@ ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xF76543210; else if (num_lanes == AVX512_NUM_SHA1_LANES) p_mgr->unused_lanes = 0xFEDCBA9876543210; +#else /* __aarch64__ */ + if (num_lanes == 1) + p_mgr->unused_lanes = 0xF0; + else if (num_lanes == 2) + p_mgr->unused_lanes = 0xF10; + else if (num_lanes == 3) + p_mgr->unused_lanes = 0xF210; + else if (num_lanes == 4) + p_mgr->unused_lanes = 0xF3210; +#endif /* __aarch64__ */ } +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 2819bcfe..04372499 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -382,12 +382,14 @@ const struct str_value_mapping hash_algo_str_map[] = { .hash_alg = TEST_HASH_CMAC } }, +#endif /* __aarch64__ */ { .name = "sha1", .values.job_params = { .hash_alg = TEST_SHA1 } }, +#ifndef __aarch64__ { .name = "sha224", .values.job_params = { diff --git a/test/kat-app/CMakeLists.txt b/test/kat-app/CMakeLists.txt index a86e6d34..7097d1d0 100644 --- a/test/kat-app/CMakeLists.txt +++ b/test/kat-app/CMakeLists.txt @@ -55,6 +55,8 @@ if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") ${CMAKE_CURRENT_SOURCE_DIR}/clear_mem_test.c ${CMAKE_CURRENT_SOURCE_DIR}/direct_api_param_test.c ${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.c + ${CMAKE_CURRENT_SOURCE_DIR}/sha_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/sha_test.json.c ) else() file(GLOB TEST_APP_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.c") diff --git a/test/kat-app/Makefile b/test/kat-app/Makefile index 9b9f77ec..4e59dbd7 100644 --- a/test/kat-app/Makefile +++ b/test/kat-app/Makefile @@ -50,7 +50,8 @@ SOURCES := main.c gcm_test.c ctr_test.c customop_test.c des_test.c ccm_test.c \ else # aarch64 SOURCES := main.c api_test.c zuc_eea3_test.c zuc_eia3_test.c snow3g_test.c direct_api_test.c \ snow3g_test_f8_vectors.json.c snow3g_test_f9_vectors.json.c clear_mem_test.c direct_api_param_test.c \ - zuc_eia3_128.json.c zuc_eia3_256.json.c zuc_eea3_128.json.c zuc_eea3_256.json.c + zuc_eia3_128.json.c zuc_eia3_256.json.c zuc_eea3_128.json.c zuc_eea3_256.json.c \ + sha_test.c sha_test.json.c endif # aarch64 OBJECTS := $(SOURCES:%.c=%.o) utils.o diff --git a/test/kat-app/main.c b/test/kat-app/main.c index e2d3fdf5..9d71abe0 100644 --- a/test/kat-app/main.c +++ b/test/kat-app/main.c @@ -159,7 +159,9 @@ struct imb_test tests[] = { { .str = "HMAC_MD5", .fn = hmac_md5_test, .enabled = 1 }, { .str = "AES", .fn = aes_test, .enabled = 1 }, { .str = "ECB", .fn = ecb_test, .enabled = 1 }, +#endif /* __aarch64__ */ { .str = "SHA", .fn = sha_test, .enabled = 1 }, +#ifndef __aarch64__ { .str = "CHAINED", .fn = chained_test, .enabled = 1 }, { .str = "HEC", .fn = hec_test, .enabled = 1 }, { .str = "AES_CBCS", .fn = aes_cbcs_test, .enabled = 1 }, diff --git a/test/kat-app/sha_test.c b/test/kat-app/sha_test.c index cb9ca3a6..000d275c 100644 --- a/test/kat-app/sha_test.c +++ b/test/kat-app/sha_test.c @@ -297,12 +297,12 @@ test_sha_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *sha1_ctx, if (!quiet_mode) printf("SHA standard test vectors (N jobs = %d):\n", num_jobs); for (; v->msg != NULL; v++) { - switch (v->tagSize) { case 160: ctx = sha1_ctx; sha_type = 1; break; +#ifndef __aarch64__ case 224: ctx = sha224_ctx; sha_type = 224; @@ -319,10 +319,15 @@ test_sha_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *sha1_ctx, ctx = sha512_ctx; sha_type = 512; break; +#endif /* __aarch64__ */ default: ctx = sha1_ctx; +#ifndef __aarch64__ printf("error #%zu, invalid tag size\n", v->tcId); test_suite_update(ctx, 0, 1); +#else /* __aarch64__ */ + test_suite_update(ctx, 0, 0); +#endif /* __aarch64__ */ continue; } #ifdef DEBUG @@ -356,19 +361,23 @@ sha_test(struct IMB_MGR *mb_mgr) unsigned i; test_suite_start(&sha1_ctx, "SHA1"); +#ifndef __aarch64__ test_suite_start(&sha224_ctx, "SHA224"); test_suite_start(&sha256_ctx, "SHA256"); test_suite_start(&sha384_ctx, "SHA384"); test_suite_start(&sha512_ctx, "SHA512"); +#endif /* __aarch64__ */ for (i = 1; i <= 17; i++) { test_sha_vectors(mb_mgr, &sha1_ctx, &sha224_ctx, &sha256_ctx, &sha384_ctx, &sha512_ctx, i); } errors = test_suite_end(&sha1_ctx); +#ifndef __aarch64__ errors += test_suite_end(&sha224_ctx); errors += test_suite_end(&sha256_ctx); errors += test_suite_end(&sha384_ctx); errors += test_suite_end(&sha512_ctx); +#endif /* __aarch64__ */ return errors; } diff --git a/test/xvalid-app/ipsec_xvalid.c b/test/xvalid-app/ipsec_xvalid.c index 1561c861..15cd0123 100644 --- a/test/xvalid-app/ipsec_xvalid.c +++ b/test/xvalid-app/ipsec_xvalid.c @@ -319,12 +319,14 @@ struct str_value_mapping hash_algo_str_map[] = { .hash_alg = IMB_AUTH_AES_CMAC_BITLEN } }, +#endif /* __aarch64__ */ { .name = "SHA1", .values.job_params = { .hash_alg = IMB_AUTH_SHA_1 } }, +#ifndef __aarch64__ { .name = "SHA224", .values.job_params = { @@ -1369,7 +1371,9 @@ fill_job(IMB_JOB *job, const struct params_s *params, uint8_t *buf, uint8_t *dig #ifndef __aarch64__ case IMB_AUTH_AES_GMAC: case IMB_AUTH_AES_CCM: +#endif /* __aarch64__ */ case IMB_AUTH_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_SHA_224: case IMB_AUTH_SHA_256: case IMB_AUTH_SHA_384: @@ -1604,7 +1608,9 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, const uint8_t *ciph case IMB_AUTH_AES_CCM: case IMB_AUTH_AES_GMAC: case IMB_AUTH_NULL: +#endif /* __aarch64__ */ case IMB_AUTH_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_SHA_224: case IMB_AUTH_SHA_256: case IMB_AUTH_SHA_384: @@ -1746,7 +1752,9 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, const uint8_t *ciph case IMB_AUTH_AES_CCM: case IMB_AUTH_AES_GMAC: case IMB_AUTH_NULL: +#endif /* __aarch64__ */ case IMB_AUTH_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_SHA_224: case IMB_AUTH_SHA_256: case IMB_AUTH_SHA_384: -- GitLab From 517b3cfc61de2c7b1777b7c91b9e0fafedb32c0e Mon Sep 17 00:00:00 2001 From: "fisher.yu" Date: Fri, 8 Nov 2024 14:05:35 +0000 Subject: [PATCH 3/3] lib: [hmac-sha1] add hmac-sha1 implementation for aarch64 platform - add single-buffer SHA1 implementation - add direct API for SHA1 - support direct/job/burst API for HMAC-SHA1 - uncomment corresponding kat/xvalid-app tests Signed-off-by: Islam Ragimov --- lib/CMakeLists.txt | 3 +- lib/Makefile | 5 +- lib/aarch64/hmac_job_neon.c | 301 ++++++++++++++++++ lib/aarch64/mb_mgr_aarch64.h | 8 + lib/aarch64/sha1_sb_aarch64_x1.S | 299 +++++++++++++++++ ..._ipad_opad_aarch64.c => sha_direct_neon.c} | 41 ++- lib/include/arch_aarch64.h | 13 + lib/include/arch_aarch64_noaesni.h | 4 + lib/include/arch_aarch64_sve256.h | 4 + lib/include/mb_mgr_burst.h | 9 +- lib/include/mb_mgr_job_api.h | 25 +- lib/include/mb_mgr_job_check.h | 4 +- lib/include/ooo_mgr_reset.h | 2 + lib/include/sha_generic.h | 36 ++- lib/x86_64/hmac_ipad_opad.c | 31 ++ lib/x86_64/ooo_mgr_reset.c | 13 + perf/ipsec_perf.c | 2 +- test/kat-app/CMakeLists.txt | 2 + test/kat-app/Makefile | 1 + test/kat-app/hmac_sha1_test.c | 12 + test/kat-app/main.c | 2 +- test/xvalid-app/ipsec_xvalid.c | 14 +- 22 files changed, 789 insertions(+), 42 deletions(-) create mode 100644 lib/aarch64/hmac_job_neon.c create mode 100644 lib/aarch64/sha1_sb_aarch64_x1.S rename lib/aarch64/{hmac_ipad_opad_aarch64.c => sha_direct_neon.c} (65%) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index c5cf05f0..3de2b16c 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -163,7 +163,8 @@ if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") "${DIR_X86_64}/snow3g_iv.c" "${DIR_X86_64}/snow3g_tables.c" "${DIR_X86_64}/ooo_mgr_reset.c" - "${DIR_X86_64}/capabilities.c") + "${DIR_X86_64}/capabilities.c" + "${DIR_X86_64}/hmac_ipad_opad.c") set(SRC_FILES_NO_AESNI "${DIR_NO_AESNI}/aesni_emu.c") else() file(GLOB SRC_FILES_AVX_T1 "${DIR_AVX_T1}/*.c") diff --git a/lib/Makefile b/lib/Makefile index eb72ed36..55b35a70 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -367,6 +367,7 @@ c_lib_objs := \ zuc_aarch64_no_aesni_top.o \ zuc_aarch64_top.o \ crc32_aarch64.o \ + sha_direct_neon.o \ sha_job_neon.o \ mb_mgr_zuc_submit_flush_aarch64.o \ mb_mgr_zuc_submit_flush_aarch64_no_aesni.o \ @@ -374,9 +375,10 @@ c_lib_objs := \ mb_mgr_snow3g_submit_flush_aarch64_no_aesni.o \ mb_mgr_snow3g_submit_flush_aarch64_sve256.o \ snow3g_aarch64_sve256.o \ + hmac_ipad_opad.o \ + hmac_job_neon.o \ cipher_suite_id.o \ ooo_mgr_reset.o \ - hmac_ipad_opad_aarch64.o \ self_test_aarch64.o \ capabilities.o asm_generic_lib_objs := \ @@ -384,6 +386,7 @@ asm_generic_lib_objs := \ snow3g_impl_aarch64_neon.o \ snow3g_impl_aarch64_sve256.o \ sha1_mb_neon_x4.o \ + sha1_sb_aarch64_x1.o \ zuc_simd.o \ zuc_simd_no_aesni.o else diff --git a/lib/aarch64/hmac_job_neon.c b/lib/aarch64/hmac_job_neon.c new file mode 100644 index 00000000..9419529d --- /dev/null +++ b/lib/aarch64/hmac_job_neon.c @@ -0,0 +1,301 @@ +/********************************************************************** + SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its + affiliates + SPDX-License-Identifier: BSD-3-Clause + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#define SHA1_MAX_JOBS NEON_NUM_SHA1_LANES + +#define SHA1_MB_IMPL sha1_mb_neon_x4 +#define SHA1_SB_IMPL sha1_sb_aarch64_x1 + +#define SUBMIT_JOB_HMAC_IMPL submit_job_hmac_neon +#define FLUSH_JOB_HMAC_IMPL flush_job_hmac_neon + + +#include "sha_generic.h" +#include "arch_aarch64.h" + +__forceinline +void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, + const unsigned lane, const int digest_row_sz) +{ + uint32_t *outp = (uint32_t *) dst; + const uint32_t *inp = (const uint32_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = bswap4(inp[digest_row_sz*lane + i]); +} + +__forceinline +void copy_bswap8_array_mb(void *dst, const void *src, const size_t num, + const unsigned lane, const int digest_row_sz) +{ + uint64_t *outp = (uint64_t *) dst; + const uint64_t *inp = (const uint64_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = bswap8(inp[digest_row_sz*lane + i]); +} + +__forceinline +void hmac_sha1_mb_init_digest(uint32_t *digest, + const unsigned lane, + const uint32_t *digest_init) +{ + digest[5*lane + 0] = digest_init[0]; + digest[5*lane + 1] = digest_init[1]; + digest[5*lane + 2] = digest_init[2]; + digest[5*lane + 3] = digest_init[3]; + digest[5*lane + 4] = digest_init[4]; +} + +__forceinline +void +hmac_sha_mb_generic_init(void *digest, const int sha_type, + const unsigned lane, const void *digest_init) +{ + if (sha_type == 1) + hmac_sha1_mb_init_digest(digest, lane, + (const uint32_t*)digest_init); +} + +__forceinline +void sha_mb_generic_write_digest(void *dst, const void *src, + const int sha_type, const unsigned lane) +{ + if (sha_type == 1) + copy_bswap4_array_mb(dst, src, NUM_SHA_DIGEST_WORDS, + lane, 5); +} + +__forceinline +void hmac_sha1_create_extra_blocks(MB_MGR_HMAC_SHA_1_OOO *state, + const uint64_t blk_size, const uint64_t r, + const unsigned min_idx) +{ + HMAC_SHA1_LANE_DATA *ld = &state->ldata[min_idx]; + const uint64_t xblk_size = blk_size*state->ldata[min_idx].extra_blocks; + + memset(ld->extra_block, 0, sizeof(ld->extra_block)); + + var_memcpy(ld->extra_block, state->args.data_ptr[min_idx], r); + ld->extra_block[r] = 0x80; + + store8_be(&ld->extra_block[xblk_size - 8], + (ld->job_in_lane->msg_len_to_hash_in_bytes + + IMB_SHA1_BLOCK_SIZE) * 8); + + state->args.data_ptr[min_idx] = &ld->extra_block[0]; + + state->lens[min_idx] = (uint16_t)xblk_size; + + state->ldata[min_idx].extra_blocks = 0; +} + +__forceinline +IMB_JOB * +submit_flush_job_hmac_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, + const unsigned max_jobs, const int is_submit, + const int sha_type, const uint64_t blk_size, + const uint64_t pad_size, + void (*mb_fn)(SHA1_ARGS *, uint32_t), + void (*sb_fn)(void * digest, const void *inp, + uint64_t num)) +{ + unsigned lane, min_idx; + IMB_JOB *ret_job = NULL; + HMAC_SHA1_LANE_DATA *ld; + + if (is_submit) { + /* + * SUBMIT + * - get a free lane id + */ + + lane = state->unused_lanes & 0xFF; + state->unused_lanes >>= 8; + state->num_lanes_inuse++; + state->args.data_ptr[lane] = + job->src + job->hash_start_src_offset_in_bytes; + + hmac_sha_mb_generic_init( + state->args.digest, + sha_type, + lane, + job->u.HMAC._hashed_auth_key_xor_ipad); + + /* copy job data in and set up initial blocks */ + state->ldata[lane].job_in_lane = job; + state->lens[lane] = job->msg_len_to_hash_in_bytes; + state->ldata[lane].extra_blocks = 1; + var_memcpy(state->ldata[lane].outer_block, + job->u.HMAC._hashed_auth_key_xor_opad, + IMB_SHA1_DIGEST_SIZE_IN_BYTES); + + /* enough jobs to start processing? */ + if (state->num_lanes_inuse != max_jobs) + return NULL; + } else { + /* + * FLUSH + * - find 1st non null job + */ + for (lane = 0; lane < max_jobs; lane++) + if (state->ldata[lane].job_in_lane != NULL) + break; + if (lane >= max_jobs) + return NULL; /* no not null job */ + } + + do { + uint64_t min_len; + unsigned i; + + if (is_submit) { + /* + * SUBMIT + * - find min common length to process + */ + min_idx = 0; + min_len = state->lens[0]; + + for (i = 1; i < max_jobs; i++) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } + } else { + /* + * FLUSH + * - copy good (not null) lane onto empty lanes + * - find min common length to process across + * - not null lanes + */ + min_idx = lane; + min_len = state->lens[lane]; + + for (i = 0; i < max_jobs; i++) { + if (i == lane) + continue; + + if (state->ldata[i].job_in_lane != NULL) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } else { + state->args.data_ptr[i] = + state->args.data_ptr[lane]; + state->lens[i] = UINT16_MAX; + } + } + } + + /* subtract min len from all lanes */ + const uint64_t min_len_blk = min_len & (~(blk_size - 1)); + + for (i = 0; i < max_jobs; i++) + state->lens[i] -= min_len_blk; + + const uint64_t r = min_len % blk_size; + + if (r >= (blk_size - pad_size)) + state->ldata[min_idx].extra_blocks = 2; + + /* run the algorithmic code on full selected blocks */ + if(min_len >= blk_size) + (*mb_fn)(&state->args, + (uint32_t)(min_len/blk_size)); + + /* create extra blocks */ + if (state->ldata[min_idx].extra_blocks != 0) + hmac_sha1_create_extra_blocks(state, blk_size, + r, min_idx); + + } while(state->lens[min_idx] != 0); + + /* outer pad */ + ld = &state->ldata[min_idx]; + ret_job = state->ldata[min_idx].job_in_lane; + + memset(ld->extra_block, 0, blk_size); + sha_mb_generic_write_digest(ld->extra_block, + state->args.digest, sha_type, + min_idx); + + ld->extra_block[IMB_SHA1_DIGEST_SIZE_IN_BYTES] = 0x80; + store8_be(&ld->extra_block[blk_size - 8], + (blk_size + IMB_SHA1_DIGEST_SIZE_IN_BYTES) * 8); + + (*sb_fn)(ld->outer_block, ld->extra_block, 1); + + copy_bswap4_array_mb(ret_job->auth_tag_output, ld->outer_block, + ret_job->auth_tag_output_len_in_bytes / 4, + 0, 0); + +#ifdef SAFE_DATA + memset(state->ldata[min_idx].extra_block, 0, sizeof(ld->extra_block)); +#endif + + /* put back processed packet into unused lanes, set job as complete */ + state->unused_lanes = (state->unused_lanes << 8) | min_idx; + state->num_lanes_inuse--; + + ret_job->status |= IMB_STATUS_COMPLETED_AUTH; + state->ldata[min_idx].job_in_lane = NULL; + return ret_job; +} + +IMB_JOB * SUBMIT_JOB_HMAC_IMPL(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) { + return submit_flush_job_hmac_sha_1(state, job, + SHA1_MAX_JOBS, /* max_jobs */ + 1, /* is_submit */ + 1, /* sha_type */ + IMB_SHA1_BLOCK_SIZE, + SHA1_PAD_SIZE, + SHA1_MB_IMPL, + SHA1_SB_IMPL); +} + +IMB_JOB * FLUSH_JOB_HMAC_IMPL(MB_MGR_HMAC_SHA_1_OOO *state) { + return submit_flush_job_hmac_sha_1(state, NULL, + SHA1_MAX_JOBS, /* max_jobs */ + 0, /* is_submit */ + 1, /* sha_type */ + IMB_SHA1_BLOCK_SIZE, + SHA1_PAD_SIZE, + SHA1_MB_IMPL, + SHA1_SB_IMPL); +} diff --git a/lib/aarch64/mb_mgr_aarch64.h b/lib/aarch64/mb_mgr_aarch64.h index 9830f654..ec15def4 100644 --- a/lib/aarch64/mb_mgr_aarch64.h +++ b/lib/aarch64/mb_mgr_aarch64.h @@ -49,6 +49,8 @@ IMB_JOB *(*SUBMIT_JOB_SHA1)(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *(*FLUSH_JOB_SHA1)(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *(*SUBMIT_JOB_HMAC)(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *(*FLUSH_JOB_HMAC)(MB_MGR_HMAC_SHA_1_OOO *state); static void reset_ooo_mgrs(IMB_MGR *state) { @@ -58,6 +60,8 @@ static void reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, NUM_ZUC_LANES); ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, NUM_ZUC_LANES); + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, NEON_NUM_SHA1_LANES); /* Init SHA1 out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, NEON_NUM_SHA1_LANES); @@ -118,8 +122,12 @@ INIT_MB_MGR_INTERNAL(IMB_MGR *state, const int reset_mgrs) state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; state->set_suite_id = SET_SUITE_ID_FN; + state->sha1_one_block = sha1_one_block_neon; + state->sha1 = sha1_neon; SUBMIT_JOB_SHA1 = submit_job_sha1_neon; FLUSH_JOB_SHA1 = flush_job_sha1_neon; + SUBMIT_JOB_HMAC = submit_job_hmac_neon; + FLUSH_JOB_HMAC = flush_job_hmac_neon; state->eea3_1_buffer = ZUC_EEA3_1_BUFFER; state->eea3_4_buffer = ZUC_EEA3_4_BUFFER; diff --git a/lib/aarch64/sha1_sb_aarch64_x1.S b/lib/aarch64/sha1_sb_aarch64_x1.S new file mode 100644 index 00000000..38a66229 --- /dev/null +++ b/lib/aarch64/sha1_sb_aarch64_x1.S @@ -0,0 +1,299 @@ +/********************************************************************** + SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its + affiliates + SPDX-License-Identifier: BSD-3-Clause + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .arch armv8-a + + digest .req x0 + input_data .req x1 + num_blocks .req w2 + + // x0 is reused intentionally between digest/tmp + // due to running out of registers + TMP .req x0 + TMPW .req w0 + sha1key_adr .req x3 + WK .req w3 + WF .req w4 + WA .req w5 + WB .req w6 + WC .req w7 + WD .req w8 + WE .req w9 + WORD0 .req w10 + WORD1 .req w11 + WORD2 .req w12 + WORD3 .req w13 + WORD4 .req w14 + WORD5 .req w15 + WORD6 .req w16 + WORD7 .req w17 + WORD8 .req w18 + WORD9 .req w19 + WORD10 .req w20 + WORD11 .req w21 + WORD12 .req w22 + WORD13 .req w23 + WORD14 .req w24 + WORD15 .req w25 + AA .req w26 + BB .req w27 + CC .req w28 + DD .req w29 + EE .req w30 + + TT .req w0 + +.macro save_stack + stp x16,x17,[sp, -128]! + stp x18,x19,[sp, 16] + stp x20,x21,[sp, 32] + stp x22,x23,[sp, 48] + stp x24,x25,[sp, 64] + stp x26,x27,[sp, 80] + stp x28,x29,[sp, 96] + str x30,[sp, 112] + // have to reuse x0, which is digest address + str x0,[sp, 120] +.endm + +.macro restore_stack + ldp x18,x19,[sp, 16] + ldp x20,x21,[sp, 32] + ldp x22,x23,[sp, 48] + ldp x24,x25,[sp, 64] + ldp x26,x27,[sp, 80] + ldp x28,x29,[sp, 96] + ldr x30,[sp, 112] + ldr x0,[sp, 120] + ldp x16,x17,[sp],128 +.endm +// macro F = (D ^ (B & (C ^ D))) +.macro FUNC_F0 + eor WF, WC, WD + and WF, WB, WF + eor WF, WD, WF +.endm + +// F = (B ^ C ^ D) +.macro FUNC_F1 + eor WF, WB, WC + eor WF, WF, WD +.endm + +// F = ((B & C) | (B & D) | (C & D)) +.macro FUNC_F2 + and TMPW, WB, WC + and WF, WB, WD + orr WF, WF, TMPW + and TMPW, WC, WD + orr WF, WF, TMPW +.endm + +// F = (B ^ C ^ D) +.macro FUNC_F3 + FUNC_F1 +.endm + +.altmacro +.macro load_next_word windex + .if \windex < 16 + load_word_at \windex + .endif +.endm + +.macro SHA1_STEP_00_15 windex:req + rev WORD\windex\(),WORD\windex\() + next_word=\windex+1 + load_next_word %next_word + + ror TMPW,WA,#32-5 + add WE,WE,TMPW + add WE,WE,WK + FUNC_F0 + ror WB,WB,#32-30 + add WE,WE,WORD\windex\() + add WE,WE,WF +.endm + +.macro SHA1_STEP_16_79 windex:req,func_f:req,reg_3:req,reg_8:req,reg_14:req,reg_16:req + eor TMPW,\reg_14,\reg_8 + eor \reg_16,\reg_16,\reg_3 + eor \reg_16,\reg_16,TMPW + + ror TMPW,WA,#32-5 + ror \reg_16,\reg_16, #32 - 1 + + add WE,WE,TMPW + add WE,WE,WK + \func_f + ror WB,WB,#32-30 + add WE,WE,\reg_16 + add WE,WE,WF +.endm + +.macro SWAP_STATES + .unreq TT + TT .req WE + .unreq WE + WE .req WD + .unreq WD + WD .req WC + .unreq WC + WC .req WB + .unreq WB + WB .req WA + .unreq WA + WA .req TT +.endm + +.altmacro +.macro SHA1_STEP_16_79_WRAPPER windex:req,func_f:req,idx3:req,idx8:req,idx14:req,idx16:req + SHA1_STEP_16_79 \windex,\func_f,WORD\idx3\(),WORD\idx8\(),WORD\idx14\(),WORD\idx16\() +.endm + +.macro exec_step windex:req + .if \windex <= 15 + SHA1_STEP_00_15 windex + .else + idx14=((\windex - 14) & 15) + idx8=((\windex - 8) & 15) + idx3=((\windex - 3) & 15) + idx16=(\windex & 15) + .if \windex <= 19 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F0,%idx3,%idx8,%idx14,%idx16 + .endif + .if \windex >= 20 && \windex <= 39 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F1,%idx3,%idx8,%idx14,%idx16 + .endif + .if \windex >= 40 && \windex <= 59 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F2,%idx3,%idx8,%idx14,%idx16 + .endif + .if \windex >= 60 && \windex <= 79 + SHA1_STEP_16_79_WRAPPER \windex,FUNC_F3,%idx3,%idx8,%idx14,%idx16 + .endif + .endif + + SWAP_STATES +.endm + +.macro exec_steps idx:req,more:vararg + exec_step \idx + .ifnb \more + exec_steps \more + .endif +.endm + +.altmacro + +.macro load_two_words_at idx0:req,idx1:req + ldp WORD\idx0\(),WORD\idx1\(),[input_data],8 +.endm + +.macro load_word_at idx:req + .if \idx % 2 == 0 + idx1=\idx+1 + load_two_words_at \idx,%idx1 + .endif +.endm + +/* + * void sha1_sb_aarch64_x1(void * digest, const void *inp, uint64_t num) + */ + .global sha1_sb_aarch64_x1 + .type sha1_sb_aarch64_x1, %function +sha1_sb_aarch64_x1: + cmp num_blocks, #0 + beq .return + + ldp WA,WB,[digest] + ldp WC,WD,[digest,8] + ldr WE,[digest,16] + save_stack + +.block_loop: + mov AA, WA + mov BB, WB + mov CC, WC + mov DD, WD + mov EE, WE + + load_word_at 0 + + adr sha1key_adr, KEY_0 + ldr WK, [sha1key_adr] + exec_steps 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19 + + // 20 ~ 39 + adr sha1key_adr, KEY_1 + ldr WK, [sha1key_adr] + exec_steps 20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 + + // 40 ~ 59 + adr sha1key_adr, KEY_2 + ldr WK, [sha1key_adr] + exec_steps 40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59 + + // 60 ~ 79 + adr sha1key_adr, KEY_3 + ldr WK, [sha1key_adr] + exec_steps 60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79 + + add WA, AA, WA + add WB, BB, WB + add WC, CC, WC + add WD, DD, WD + add WE, EE, WE + + subs num_blocks, num_blocks, 1 + bne .block_loop + + restore_stack + stp WA,WB,[digest] + stp WC,WD,[digest,8] + str WE,[digest,16] + +.return: + ret + + .size sha1_sb_aarch64_x1, .-sha1_sb_aarch64_x1 + +.align 4 +KEY_0: + .word 0x5a827999 +KEY_1: + .word 0x6ed9eba1 +KEY_2: + .word 0x8f1bbcdc +KEY_3: + .word 0xca62c1d6 diff --git a/lib/aarch64/hmac_ipad_opad_aarch64.c b/lib/aarch64/sha_direct_neon.c similarity index 65% rename from lib/aarch64/hmac_ipad_opad_aarch64.c rename to lib/aarch64/sha_direct_neon.c index 545526c6..8727672a 100644 --- a/lib/aarch64/hmac_ipad_opad_aarch64.c +++ b/lib/aarch64/sha_direct_neon.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2024 Arm Corporation All rights reserved. + Copyright(c) 2023 Arm Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -26,23 +26,30 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ -#include -#include -#include +#define EXTENSION_SHA1 ARCH_NEON -IMB_DLL_EXPORT -__attribute((noreturn)) void -imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, - const void *pkey, const size_t key_len, - void *ipad_hash, void *opad_hash) +#define SHA1_ONE_BLOCK_IMPL sha1_one_block_neon +#define SHA1_IMPL sha1_neon + +#include "include/sha_generic.h" +#include "include/arch_aarch64.h" + +/* ========================================================================== */ +/* One block SHA1 computation for IPAD / OPAD usage only */ + +void SHA1_ONE_BLOCK_IMPL(const void *data, void *digest) +{ + sha_generic_1block(data, digest, EXTENSION_SHA1, 1 /* SHA1 */); +} + +/* ========================================================================== */ +/* + * SHA1 API for use in HMAC-SHA1 when key is longer than the block size + */ + +void SHA1_IMPL(const void *data, const uint64_t length, void *digest) { - /* - * This method isn't supported on AArch64. However, it is stubbed to - * ensure ABI compatibility with the x86_64 ipsec-mb library. - * - * As this should never be called, it crashes. - */ - fprintf(stderr, "imb_hac_ipad_opad() is unimplemented for AArch64.\n"); - abort(); + sha_generic(data, length, digest, EXTENSION_SHA1, 1, IMB_SHA1_BLOCK_SIZE, + SHA1_PAD_SIZE); } diff --git a/lib/include/arch_aarch64.h b/lib/include/arch_aarch64.h index 08411344..ddbd54d3 100644 --- a/lib/include/arch_aarch64.h +++ b/lib/include/arch_aarch64.h @@ -63,6 +63,10 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_aarch64 #define FLUSH_JOB_SHA1 flush_job_sha1_aarch64 +/* HMAC-SHA1 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_aarch64 +#define FLUSH_JOB_HMAC flush_job_hmac_aarch64 + /* ZUC/ZUC256 */ #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 @@ -137,6 +141,15 @@ uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); /* SHA */ void sha1_mb_neon_x4(SHA1_ARGS *args, uint32_t size_in_blocks); +void sha1_sb_aarch64_x1(void * digest, const void *inp, uint64_t num); + +IMB_DLL_EXPORT void sha1_neon(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha1_one_block_neon(const void *data, void *digest); + +IMB_JOB *submit_job_hmac_neon(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_neon(MB_MGR_HMAC_SHA_1_OOO *state); IMB_JOB *submit_job_sha1_neon(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); diff --git a/lib/include/arch_aarch64_noaesni.h b/lib/include/arch_aarch64_noaesni.h index aef53c9e..a9c89374 100644 --- a/lib/include/arch_aarch64_noaesni.h +++ b/lib/include/arch_aarch64_noaesni.h @@ -58,6 +58,10 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_aarch64_no_aesni #define FLUSH_JOB_SHA1 flush_job_sha1_aarch64_no_aesni +/* HMAC-SHA1 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_aarch64_no_aesni +#define FLUSH_JOB_HMAC flush_job_hmac_aarch64_no_aesni + /* ZUC/ZUC256 */ #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_no_aesni #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_no_aesni diff --git a/lib/include/arch_aarch64_sve256.h b/lib/include/arch_aarch64_sve256.h index f5fafd1f..4f8b1932 100644 --- a/lib/include/arch_aarch64_sve256.h +++ b/lib/include/arch_aarch64_sve256.h @@ -58,6 +58,10 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_aarch64_sve256 #define FLUSH_JOB_SHA1 flush_job_sha1_aarch64_sve256 +/* HMAC-SHA1 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_aarch64_sve256 +#define FLUSH_JOB_HMAC flush_job_hmac_aarch64_sve256 + /* ZUC/ZUC256 */ #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index 361ec1f1..2d0f3a68 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -566,6 +566,7 @@ SUBMIT_AEAD_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, { return submit_aead_burst_and_check(state, jobs, n_jobs, cipher, dir, key_size, 0); } +#endif /* __aarch64__ */ __forceinline uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const int run_check, @@ -607,7 +608,9 @@ submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, co completed_jobs++; } } - } else if (hash_alg == IMB_AUTH_HMAC_SHA_224) { + } +#ifndef __aarch64__ + else if (hash_alg == IMB_AUTH_HMAC_SHA_224) { /* submit all jobs */ for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; @@ -688,10 +691,10 @@ submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, co } } } +#endif /* __aarch64__ */ return completed_jobs; } -#endif /* __aarch64__ */ __forceinline uint32_t submit_burst_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const int run_check, @@ -836,9 +839,9 @@ submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs } switch (hash) { -#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_1); +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_224); diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index 404c8c90..275ddc5b 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -2648,8 +2648,8 @@ static const submit_flush_fn_t tab_flush_cipher[] = { __forceinline IMB_JOB * SUBMIT_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) { -#ifndef __aarch64__ MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; +#ifndef __aarch64__ MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; @@ -2681,9 +2681,9 @@ SUBMIT_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) #endif /* __aarch64__ */ switch (hash_alg) { -#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: return SUBMIT_JOB_HMAC(hmac_sha_1_ooo, job); +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: return SUBMIT_JOB_HMAC_SHA_224(hmac_sha_224_ooo, job); case IMB_AUTH_HMAC_SHA_256: @@ -2851,8 +2851,8 @@ SUBMIT_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) __forceinline IMB_JOB * FLUSH_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) { -#ifndef __aarch64__ MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; +#ifndef __aarch64__ MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; @@ -2884,9 +2884,9 @@ FLUSH_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) #endif /* __aarch64__ */ switch (hash_alg) { -#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: return FLUSH_JOB_HMAC(hmac_sha_1_ooo); +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: return FLUSH_JOB_HMAC_SHA_224(hmac_sha_224_ooo); case IMB_AUTH_HMAC_SHA_256: @@ -2958,13 +2958,13 @@ FLUSH_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) /* Generate specialized hash submit functions and create a table */ /* ========================================================================= */ -#ifndef __aarch64__ static IMB_JOB * submit_hash_hmac_sha1(IMB_MGR *state, IMB_JOB *job) { return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_1); } +#ifndef __aarch64__ static IMB_JOB * submit_hash_hmac_sha224(IMB_MGR *state, IMB_JOB *job) { @@ -3261,9 +3261,9 @@ submit_hash_hmac_sm3(IMB_MGR *state, IMB_JOB *job) static const submit_flush_fn_t tab_submit_hash[] = { /* [0] invalid entry */ NULL, -#ifndef __aarch64__ /* [1] HMAC-SHA1 */ submit_hash_hmac_sha1, +#ifndef __aarch64__ /* [2] HMAC-SHA224 */ submit_hash_hmac_sha224, /* [3] HMAC-SHA256 */ @@ -3277,9 +3277,9 @@ static const submit_flush_fn_t tab_submit_hash[] = { /* [7] HMAC-MD5 */ submit_hash_hmac_md5, #else /* __aarch64__ */ - /* [1] - [7] NULL */ + /* [2] - [7] NULL */ NULL, NULL, NULL, NULL, - NULL, NULL, NULL, + NULL, NULL, #endif /* __aarch64__ */ /* [8] NULL */ submit_hash_null, @@ -3400,13 +3400,14 @@ static const submit_flush_fn_t tab_submit_hash[] = { /* ========================================================================= */ /* Generate specialized hash flush functions and create a table */ /* ========================================================================= */ -#ifndef __aarch64__ + static IMB_JOB * flush_hash_hmac_sha1(IMB_MGR *state, IMB_JOB *job) { return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_1); } +#ifndef __aarch64__ static IMB_JOB * flush_hash_hmac_sha224(IMB_MGR *state, IMB_JOB *job) { @@ -3703,9 +3704,9 @@ flush_hash_hmac_sm3(IMB_MGR *state, IMB_JOB *job) static const submit_flush_fn_t tab_flush_hash[] = { /* [0] invalid entry */ NULL, -#ifndef __aarch64__ /* [1] HMAC-SHA1 */ flush_hash_hmac_sha1, +#ifndef __aarch64__ /* [2] HMAC-SHA224 */ flush_hash_hmac_sha224, /* [3] HMAC-SHA256 */ @@ -3719,9 +3720,9 @@ static const submit_flush_fn_t tab_flush_hash[] = { /* [7] HMAC-MD5 */ flush_hash_hmac_md5, #else /* __aarch64__ */ - /* [1] - [7] NULL */ + /* [2] - [7] NULL */ NULL, NULL, NULL, NULL, - NULL, NULL, NULL, + NULL, NULL, #endif /* __aarch64__ */ /* [8] NULL */ flush_hash_null, diff --git a/lib/include/mb_mgr_job_check.h b/lib/include/mb_mgr_job_check.h index 268936aa..59741054 100644 --- a/lib/include/mb_mgr_job_check.h +++ b/lib/include/mb_mgr_job_check.h @@ -1235,13 +1235,14 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, const IMB_CIPHER_MODE cipher_ } switch (hash_alg) { -#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_MD5: case IMB_AUTH_HMAC_SHA_224: case IMB_AUTH_HMAC_SHA_256: case IMB_AUTH_HMAC_SHA_384: case IMB_AUTH_HMAC_SHA_512: +#endif /* __aarch64__ */ if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); return 1; @@ -1269,6 +1270,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, const IMB_CIPHER_MODE cipher_ return 1; } break; +#ifndef __aarch64__ case IMB_AUTH_AES_XCBC: if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index 7b153398..a701172d 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -44,11 +44,13 @@ ooo_mgr_ccm_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes); +#endif /* __aarch64__ */ IMB_DLL_LOCAL void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes); diff --git a/lib/include/sha_generic.h b/lib/include/sha_generic.h index 85c917ba..9f9d5164 100644 --- a/lib/include/sha_generic.h +++ b/lib/include/sha_generic.h @@ -34,14 +34,27 @@ #include "ipsec_ooo_mgr.h" #include "constants.h" +#ifndef __aarch64__ #include "include/clear_regs_mem.h" +#else /* __aarch64__ */ +#include "aarch64/clear_regs_mem_aarch64.h" +#endif #include "include/error.h" #include "include/arch_sse_type1.h" #include "include/arch_sse_type2.h" #include "include/arch_avx_type1.h" #include "include/arch_avx2_type4.h" -enum arch_type { ARCH_SSE = 0, ARCH_SSE_SHANI, ARCH_AVX, ARCH_AVX2_SHANI }; +extern void +sha1_sb_aarch64_x1(void * digest, const void *inp, uint64_t num); + +enum arch_type { +#ifndef __aarch64__ + ARCH_SSE = 0, ARCH_SSE_SHANI, ARCH_AVX, ARCH_AVX2_SHANI +#else /* __aarch64__ */ + ARCH_NEON = 0, ARCH_CE, +#endif /* __aarch64__ */ +}; /* ========================================================================== */ /* @@ -109,6 +122,7 @@ copy_bswap8_array(void *dst, const void *src, const size_t num) __forceinline void sha_generic_one_block(const void *inp, void *digest, const enum arch_type arch, const int sha_type) { +#ifndef __aarch64__ if (sha_type == 1) { IMB_ASSERT(arch != ARCH_AVX2_SHANI); if (arch == ARCH_AVX) @@ -160,6 +174,16 @@ sha_generic_one_block(const void *inp, void *digest, const enum arch_type arch, sha512_block_sse(inp, digest); #endif } +#else /* __aarch64__ */ + switch (sha_type) { + case 1: + sha1_sb_aarch64_x1(digest, inp, 1); + break; + default: + break; + } + +#endif /* __aarch64__ */ } __forceinline void @@ -313,11 +337,16 @@ sha_generic(const void *data, const uint64_t length, void *digest, const enum ar #ifdef SAFE_DATA clear_mem(cb, sizeof(cb)); clear_mem(&local_digest, sizeof(local_digest)); +#ifndef __aarch64__ clear_scratch_gps(); if (arch == ARCH_AVX || arch == ARCH_AVX2_SHANI) clear_scratch_xmms_avx(); else clear_scratch_xmms_sse(); +#else /* __aarch64__ */ + CLEAR_SCRATCH_GPS(); + CLEAR_SCRATCH_SIMD_REGS(); +#endif /* __aarch64__ */ #endif } @@ -338,11 +367,16 @@ sha_generic_1block(const void *data, void *digest, const enum arch_type arch, co sha_generic_init(digest, sha_type); sha_generic_one_block(data, digest, arch, sha_type); #ifdef SAFE_DATA +#ifndef __aarch64__ clear_scratch_gps(); if (arch == ARCH_AVX || arch == ARCH_AVX2_SHANI) clear_scratch_xmms_avx(); else clear_scratch_xmms_sse(); +#else /* __aarch64__ */ + CLEAR_SCRATCH_GPS(); + CLEAR_SCRATCH_SIMD_REGS(); +#endif /* __aarch64__ */ #endif } diff --git a/lib/x86_64/hmac_ipad_opad.c b/lib/x86_64/hmac_ipad_opad.c index f92e1ae4..4c4272f2 100644 --- a/lib/x86_64/hmac_ipad_opad.c +++ b/lib/x86_64/hmac_ipad_opad.c @@ -32,7 +32,11 @@ #include #include "include/error.h" + +#ifndef __aarch64__ #include "include/memcpy.h" +#endif + #include "include/arch_sse_type1.h" /* sm3_one_block_sse(), sm3_msg_sse() */ IMB_DLL_EXPORT @@ -59,6 +63,7 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke local_key_len = (key_len <= IMB_SHA1_BLOCK_SIZE) ? key_len : IMB_SHA1_DIGEST_SIZE_IN_BYTES; break; +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: local_key_len = (key_len <= IMB_SHA_224_BLOCK_SIZE) ? key_len @@ -94,6 +99,7 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke case IMB_AUTH_HMAC_SM3: local_key_len = (key_len <= IMB_SM3_BLOCK_SIZE) ? key_len : IMB_SM3_DIGEST_SIZE; break; +#endif /* __aarch64__ */ default: imb_set_errno(NULL, IMB_ERR_HASH_ALGO); return; @@ -103,12 +109,17 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke /* prepare the key */ if (local_key_len == key_len) { +#ifndef __aarch64__ safe_memcpy(key, pkey, key_len); +#else /* __aarhc64 */ + memcpy(key, pkey, key_len); +#endif /* __aarch64__ */ } else switch (sha_type) { case IMB_AUTH_HMAC_SHA_1: IMB_SHA1(mb_mgr, pkey, key_len, key); break; +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: IMB_SHA224(mb_mgr, pkey, key_len, key); break; @@ -124,6 +135,12 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke default: /* For SHA-512 */ IMB_SHA512(mb_mgr, pkey, key_len, key); } +#else /* __aarch64__ */ + default: + imb_set_errno(NULL, IMB_ERR_HASH_ALGO); + return; + } +#endif /* __aarch64__ */ /* compute ipad hash */ if (ipad_hash != NULL) { @@ -134,6 +151,7 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke case IMB_AUTH_HMAC_SHA_1: IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); break; +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); break; @@ -152,6 +170,12 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke default: /* For MD5*/ IMB_MD5_ONE_BLOCK(mb_mgr, buf, ipad_hash); } +#else /* __aarch64__ */ + default: + imb_set_errno(NULL, IMB_ERR_HASH_ALGO); + return; + } +#endif /* __aarch64__ */ } /* compute opad hash */ @@ -163,6 +187,7 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke case IMB_AUTH_HMAC_SHA_1: IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); break; +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); break; @@ -181,6 +206,12 @@ imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pke default: /* For MD5 */ IMB_MD5_ONE_BLOCK(mb_mgr, buf, opad_hash); } +#else /* __aarch64__ */ + default: + imb_set_errno(NULL, IMB_ERR_HASH_ALGO); + return; + } +#endif /* __aarch64__ */ } #ifdef SAFE_DATA diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index d8fd4155..1c909ebc 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -128,6 +128,7 @@ ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; } +#endif /* __aarch64__ */ IMB_DLL_LOCAL void @@ -150,6 +151,7 @@ ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) IMB_ASSERT(AVX_NUM_SHA1_LANES == SSE_NUM_SHA1_LANES); +#ifndef __aarch64__ if (num_lanes == 2) p_mgr->unused_lanes = 0xFF0100; /* SHANI */ else if (num_lanes == AVX_NUM_SHA1_LANES) @@ -158,8 +160,19 @@ ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xF76543210; else if (num_lanes == AVX512_NUM_SHA1_LANES) p_mgr->unused_lanes = 0xFEDCBA9876543210; +#else /* __aarch64__ */ + if (num_lanes == 1) + p_mgr->unused_lanes = 0xFF00; + else if (num_lanes == 2) + p_mgr->unused_lanes = 0xFF0100; + else if (num_lanes == 3) + p_mgr->unused_lanes = 0xFF020100; + else if (num_lanes == 4) + p_mgr->unused_lanes = 0xFF03020100; +#endif /* __aarch64__ */ } +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 04372499..2e05183a 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -333,13 +333,13 @@ const struct str_value_mapping cipher_algo_str_map[] = { }; const struct str_value_mapping hash_algo_str_map[] = { -#ifndef __aarch64__ { .name = "sha1-hmac", .values.job_params = { .hash_alg = TEST_SHA1_HMAC } }, +#ifndef __aarch64__ { .name = "sha224-hmac", .values.job_params = { diff --git a/test/kat-app/CMakeLists.txt b/test/kat-app/CMakeLists.txt index 7097d1d0..0396b749 100644 --- a/test/kat-app/CMakeLists.txt +++ b/test/kat-app/CMakeLists.txt @@ -55,6 +55,8 @@ if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") ${CMAKE_CURRENT_SOURCE_DIR}/clear_mem_test.c ${CMAKE_CURRENT_SOURCE_DIR}/direct_api_param_test.c ${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha1_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha1.json.c ${CMAKE_CURRENT_SOURCE_DIR}/sha_test.c ${CMAKE_CURRENT_SOURCE_DIR}/sha_test.json.c ) diff --git a/test/kat-app/Makefile b/test/kat-app/Makefile index 4e59dbd7..23fef133 100644 --- a/test/kat-app/Makefile +++ b/test/kat-app/Makefile @@ -51,6 +51,7 @@ else # aarch64 SOURCES := main.c api_test.c zuc_eea3_test.c zuc_eia3_test.c snow3g_test.c direct_api_test.c \ snow3g_test_f8_vectors.json.c snow3g_test_f9_vectors.json.c clear_mem_test.c direct_api_param_test.c \ zuc_eia3_128.json.c zuc_eia3_256.json.c zuc_eea3_128.json.c zuc_eea3_256.json.c \ + hmac_sha1_test.c hmac_sha1.json.c \ sha_test.c sha_test.json.c endif # aarch64 diff --git a/test/kat-app/hmac_sha1_test.c b/test/kat-app/hmac_sha1_test.c index 4fea42cb..8a71ffdb 100644 --- a/test/kat-app/hmac_sha1_test.c +++ b/test/kat-app/hmac_sha1_test.c @@ -135,10 +135,22 @@ test_hmac_sha1(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_ * SHANI HMAC-SHA implementation can return a completed * job after 2nd submission */ +#ifndef __aarch64__ if (num_jobs < 2) { printf("%d Unexpected return from submit_job\n", __LINE__); goto end; } +#else + /* + * The condition check above is specific to the Intel + * implementation. This condition does not hold + * for the AARCH64 implementation, because: + * When crypto extension is unavailable, single buffer + * is implemented for AARCH64. + * Single buffer implementation returns result after + * first submit call. + */ +#endif if (!hmac_sha1_job_ok(vec, job, job->user_data, padding, sizeof(padding))) goto end; } diff --git a/test/kat-app/main.c b/test/kat-app/main.c index 9d71abe0..65e630a7 100644 --- a/test/kat-app/main.c +++ b/test/kat-app/main.c @@ -153,8 +153,8 @@ struct imb_test tests[] = { { .str = "KASUMI", .fn = kasumi_test, .enabled = 1 }, #endif /* __aarch64__ */ { .str = "SNOW3G", .fn = snow3g_test, .enabled = 1 }, -#ifndef __aarch64__ { .str = "HMAC_SHA1", .fn = hmac_sha1_test, .enabled = 1 }, +#ifndef __aarch64__ { .str = "HMAC_SHA256", .fn = hmac_sha256_sha512_test, .enabled = 1 }, { .str = "HMAC_MD5", .fn = hmac_md5_test, .enabled = 1 }, { .str = "AES", .fn = aes_test, .enabled = 1 }, diff --git a/test/xvalid-app/ipsec_xvalid.c b/test/xvalid-app/ipsec_xvalid.c index 15cd0123..aad04b6a 100644 --- a/test/xvalid-app/ipsec_xvalid.c +++ b/test/xvalid-app/ipsec_xvalid.c @@ -256,13 +256,13 @@ struct str_value_mapping cipher_algo_str_map[] = { }; struct str_value_mapping hash_algo_str_map[] = { -#ifndef __aarch64__ { .name = "HMAC-SHA1", .values.job_params = { .hash_alg = IMB_AUTH_HMAC_SHA_1 } }, +#ifndef __aarch64__ { .name = "HMAC-SHA224", .values.job_params = { @@ -1317,18 +1317,20 @@ fill_job(IMB_JOB *job, const struct params_s *params, uint8_t *buf, uint8_t *dig job->u.CMAC._skey1 = k2; job->u.CMAC._skey2 = k3; break; +#endif /* __aarch64__ */ case IMB_AUTH_HMAC_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: case IMB_AUTH_HMAC_SHA_256: case IMB_AUTH_HMAC_SHA_384: case IMB_AUTH_HMAC_SHA_512: case IMB_AUTH_HMAC_SM3: case IMB_AUTH_MD5: +#endif /* __aarch64__ */ /* HMAC hash alg is SHA1 or MD5 */ job->u.HMAC._hashed_auth_key_xor_ipad = (uint8_t *) ipad; job->u.HMAC._hashed_auth_key_xor_opad = (uint8_t *) opad; break; -#endif /* __aarch64__ */ case IMB_AUTH_ZUC256_EIA3_BITLEN: job->u.ZUC_EIA3._key = k2; if (auth_iv_size == 23) { @@ -1585,17 +1587,19 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, const uint8_t *ciph case IMB_AUTH_POLY1305: nosimd_memset(k1_expanded, pattern_auth_key, sizeof(keys->k1_expanded)); break; +#endif /* __aarch64__ */ case IMB_AUTH_HMAC_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: case IMB_AUTH_HMAC_SHA_256: case IMB_AUTH_HMAC_SHA_384: case IMB_AUTH_HMAC_SHA_512: case IMB_AUTH_HMAC_SM3: case IMB_AUTH_MD5: +#endif /* __aarch64__ */ nosimd_memset(ipad, pattern_auth_key, sizeof(keys->ipad)); nosimd_memset(opad, pattern_auth_key, sizeof(keys->opad)); break; -#endif /* __aarch64__ */ case IMB_AUTH_ZUC_EIA3_BITLEN: case IMB_AUTH_ZUC256_EIA3_BITLEN: case IMB_AUTH_SNOW3G_UIA2_BITLEN: @@ -1718,16 +1722,18 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, const uint8_t *ciph IMB_AES_KEYEXP_256(mb_mgr, auth_key, k1_expanded, dust); IMB_AES_CMAC_SUBKEY_GEN_256(mb_mgr, k1_expanded, k2, k3); break; +#endif /* __aarch64__ */ case IMB_AUTH_HMAC_SHA_1: +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_224: case IMB_AUTH_HMAC_SHA_256: case IMB_AUTH_HMAC_SHA_384: case IMB_AUTH_HMAC_SHA_512: case IMB_AUTH_HMAC_SM3: case IMB_AUTH_MD5: +#endif /* __aarch64__ */ imb_hmac_ipad_opad(mb_mgr, params->hash_alg, auth_key, MAX_KEY_SIZE, ipad, opad); break; -#endif /* __aarch64__ */ case IMB_AUTH_ZUC_EIA3_BITLEN: case IMB_AUTH_ZUC256_EIA3_BITLEN: case IMB_AUTH_SNOW3G_UIA2_BITLEN: -- GitLab