diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000000000000000000000000000000000..194dba31c6dfcee38afee6516173d212a4936690 --- /dev/null +++ b/.clang-format @@ -0,0 +1,47 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +BasedOnStyle: LLVM +IndentWidth: 8 +Language: Cpp +BreakBeforeBraces: Linux +AllowShortIfStatementsOnASingleLine: false +IndentCaseLabels: false +UseTab: Never +AlignConsecutiveMacros: true +AlignTrailingComments: true +AlwaysBreakAfterReturnType: All +SortIncludes: false +BreakBeforeInheritanceComma: true +AllowAllParametersOfDeclarationOnNextLine: false +BinPackParameters: true +BinPackArguments: true +ReflowComments: true +SpaceAfterCStyleCast: false +ColumnLimit: 100 +Cpp11BracedListStyle: false +MaxEmptyLinesToKeep: 1 +ContinuationIndentWidth: 8 +SpaceAfterCStyleCast: true diff --git a/.gitignore b/.gitignore index 24e1c5d1970968679002f804ad7eb4132331094c..c357b2711f92d573cf35ac2b010452e0bf1c102b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,12 +10,17 @@ *.d *.dep *~ -ipsec_MB_testapp* -ipsec_perf* -ipsec_xvalid_test* -job_api_fuzz_test* +imb-perf* +imb-kat* +imb-xvalid* +imb-fuzz-* +imb-acvp* +imb-wycheproof* +imb-burst* *_lnk.def TAGS -acvp_app -test/wycheproof/wycheproof !wycheproof.c +!ipsec_perf.c +.cppcheck +.bughunt +/docs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 319950dc9d2560cac1c222129ef45e2f95367fba..3affcbf752be171abb999ae2103ed85054cc935a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ unit-test-job:aarch64: # This job runs in the test stage. script: - echo "Running unit tests..." - export LD_LIBRARY_PATH=./lib - - ./test/ipsec_MB_testapp + - ./test/kat-app/imb-kat - echo "Unit tests complete" dependencies: - build-job:aarch64 @@ -57,7 +57,7 @@ xvalid-test-job:aarch64: # This job also runs in the test stage. script: - echo "Running xvalid test." - export LD_LIBRARY_PATH=./lib - - ./test/ipsec_xvalid_test + - ./test/xvalid-app/imb-xvalid - echo "xvalid test complete." dependencies: - build-job:aarch64 @@ -70,7 +70,7 @@ unit-test-job:x86_64: # This job runs in the test stage. script: - echo "Running unit tests..." - export LD_LIBRARY_PATH=./lib - - ./test/ipsec_MB_testapp + - ./test/kat-app/imb-kat - echo "Unit tests complete" dependencies: - build-job:x86_64 @@ -83,7 +83,7 @@ xvalid-test-job:x86_64: # This job also runs in the test stage. script: - echo "Running xvalid test." - export LD_LIBRARY_PATH=./lib - - ./test/ipsec_xvalid_test + - ./test/xvalid-app/imb-xvalid - echo "xvalid test complete." dependencies: - build-job:x86_64 diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d137577b4204a5c4106faa373d6fb77cb05d5a96 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,97 @@ +# Copyright (c) 2022-2023, Intel Corporation +# Copyright 2000-2023 Kitware, Inc. and Contributors +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake") + +# get version from public header file +imb_get_version("${CMAKE_CURRENT_SOURCE_DIR}/lib/ipsec-mb.h") +message(STATUS "Project Version: ${IPSEC_MB_VERSION_FULL}") + +cmake_minimum_required(VERSION 3.16) + +# set default project values +imb_set_proj_defaults() + +project(ipsec-mb VERSION ${IPSEC_MB_VERSION} + DESCRIPTION "IPsec Multi-Buffer library") + +# add testing support +include(CTest) + +# set binary output directory if specified +imb_set_binary_dir() + +# detect OS +imb_detect_os() + +# do compiler checks +imb_compiler_check() + +####################################### +# configure pre gen custom targets +####################################### + +# add custom target to uninstall library +imb_add_target_uninstall("${CMAKE_CURRENT_SOURCE_DIR}/lib/cmake/uninstall.cmake.in") + +# style check target +imb_add_target_style_check() + +# TAGS target +imb_add_target_tags() + +# spellcheck target +imb_add_target_spellcheck() + +# cppcheck and bughunt targets +imb_add_target_cppcheck_bughunt() + +# doxy target to generate documentation +imb_add_target_doxy() + +####################################### +# configure main project targets +####################################### + +# build library +add_subdirectory(lib) + +# build perf application +add_subdirectory(perf) + +# build test applications +add_subdirectory(test) + +# build example applications +add_subdirectory(examples) + +####################################### +# configure post gen custom targets +####################################### + +# add custom target to print help information +imb_add_target_print_help("${IPSEC_MB_OPTIONS}") + diff --git a/CONTRIBUTING b/CONTRIBUTING index 2ad914c09f4d5f7ad9e85e190484ec3e3a6579cc..5b78bd5b9ab4371403ef07710015ec9047ef1daf 100755 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -7,7 +7,8 @@ to significant code or documentation features. Note: There is just one branch used in the project. All development is done on the -main branch. +main branch. Code taken from the tip of the main branch should not be +considered fit for production. How can I contribute? diff --git a/LICENSE b/LICENSE index aaf3c457830d060a8fd91aa1a36642d64fb1eda1..a0f261ffa79351a4a0e29891272a42f5e63853d6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,4 @@ -Copyright (c) 2012-2022, Intel Corporation -Portions Copyright (c) 2021-2022, IPSec-MB contributors (where specified in -individual files) +Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Makefile b/Makefile index 9da44c69c48686061ad500f17fb14406ff6c1efb..6915451248e55f75ad5f584b81626cf8691eb568 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022, Intel Corporation +# Copyright (c) 2020-2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -30,22 +30,20 @@ all: $(MAKE) -C lib $(MAKE) -C test -ifeq ($(ARCH),x86_64) - $(MAKE) -C test/wycheproof -endif $(MAKE) -C perf + $(MAKE) -C examples clean: $(MAKE) -C lib clean $(MAKE) -C test clean - $(MAKE) -C test/wycheproof clean $(MAKE) -C perf clean + $(MAKE) -C examples clean style: $(MAKE) -C lib style $(MAKE) -C test style - $(MAKE) -C test/wycheproof style $(MAKE) -C perf style + $(MAKE) -C examples style install: $(MAKE) -C lib install @@ -79,13 +77,21 @@ CS_IGNORE_WORDS ?= iinclude,struc,fo,ue,od,ba,padd .PHONY: spellcheck spellcheck: $(CODESPELL) -d -L $(CS_IGNORE_WORDS) \ - -S "*.obj,*.o,*.a,*.so,*.lib,*~,*.so,*.so.*,*.d,ipsec_perf" \ - -S "ipsec_MB_testapp,ipsec_xvalid_test" \ + -S "*.obj,*.o,*.a,*.so,*.lib,*~,*.so,*.so.*,*.d,imb-perf" \ + -S "imb-kat,imb-xvalid" \ ./lib ./perf ./test README.md SECURITY.md CONTRIBUTING \ Makefile win_x64.mak ReleaseNotes.txt LICENSE $(CS_EXTRA_OPTS) # cppcheck analysis check -CPPCHECK ?= cppcheck -.PHONY: cppcheck cppcheck: - $(CPPCHECK) ./ --force --enable=all + $(MAKE) -C lib cppcheck + $(MAKE) -C test cppcheck + $(MAKE) -C perf cppcheck + $(MAKE) -C examples cppcheck + +# cppcheck bughunt analysis check +bughunt: + $(MAKE) -C lib bughunt + $(MAKE) -C test bughunt + $(MAKE) -C perf bughunt + $(MAKE) -C examples bughunt diff --git a/README.md b/README.md index 16c7987eefada31f3a44fd9d7836d3b8cc867571..c02b50a01563e34aba11723184807dabe4354ac8 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,24 @@ +[![Coverity Status](https://scan.coverity.com/projects/16449/badge.svg)](https://scan.coverity.com/projects/intel-ipsec-mb) +![Linux Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_shared_gcc.yml/badge.svg) +![Linux Build Static gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_static_gcc.yml/badge.svg) +![Linux Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_shared_clang.yml/badge.svg) +![Linux Build Static clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_static_clang.yml/badge.svg) +![Linux Build Shared clang AESNI emulation](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_shared_clang_aesni_emu.yml/badge.svg) +![FreeBSD Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_build_shared_clang.yml/badge.svg) +![FreeBSD Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_build_shared_gcc.yml/badge.svg) + # Multi-Buffer Crypto for IPsec Library The code in this repository is based on https://github.com/intel/intel-ipsec-mb, and is regularly kept up-to-date with that project. Further enhancements have been added as part of development in this repository, particularly with regard to optimisations -for Arm AArch64 systems. The library provides software crypto acceleration primarily -targeting packet processing applications. It can be used for application such as: -IPsec, TLS, Wireless (RAN), Cable or MPEG DRM. +for Arm AArch64 systems. +The library provides software crypto acceleration primarily targeting packet processing +applications. It can be used for application such as: IPsec, TLS, Wireless (RAN), Cable or MPEG DRM. -The library can be used as software crypto provider in [DPDK](https://www.dpdk.org/), -[Intel(R) QAT Engine](https://github.com/intel/QAT_Engine) and [FD.io](https://fd.io/). +The library is hosted on [GitLab](https://gitlab.arm.com/arm-reference-solutions/ipsec-mb) and is used as software crypto +provider in [DPDK](https://www.dpdk.org/), [Intel(R) QAT Engine](https://github.com/intel/QAT_Engine) +and [FD.io](https://fd.io/). Using crypto interfaces from the above frameworks gives freedom to change providers without subsequent application modifications. The library can also be used directly @@ -23,20 +33,24 @@ Key differentiating features: - multi-buffer and function stitching innovations - low level implementations using latest instruction extensions -Contents -======== +## Contents + 1. Overview 2. Processor Extensions 3. Recommendations 4. Package Content -5. Compilation -6. Security Considerations & Options for Increased Security -7. Installation -8. Backwards compatibility -9. Disclaimer (ZUC, KASUMI, SNOW3G) - -1\. Overview -============ +5. Documentation +6. Compilation +7. Security Considerations & Options for Increased Security +8. Installation +9. Backwards compatibility +10. Disclaimer (ZUC, KASUMI, SNOW3G) +11. Legal Disclaimer +12. FIPS Compliance +13. DLL Injection + +## 1. Overview + Multi-Buffer Crypto for IPsec Library is highly-optimized software implementations of the core cryptographic processing for IPsec, which provides industry-leading performance on a range of Intel(R) and Arm @@ -56,9 +70,9 @@ Table 1. List of supported cipher algorithms and their implementations. | Encryption +---------------------------------------------------------------| | | x86_64 | SSE | AVX | AVX2 | AVX512 | VAES(5)| aarch64 | |----------------+--------+--------+--------+--------+--------+--------|---------| -| AES128-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | -| AES192-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | -| AES256-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | +| AES128-GCM | N | Y by8 | N | Y by8 | Y by8 | Y by32 | N | +| AES192-GCM | N | Y by8 | N | Y by8 | Y by8 | Y by32 | N | +| AES256-GCM | N | Y by8 | N | Y by8 | Y by8 | Y by32 | N | | AES128-CCM | N | Y by8 | Y by8 | N | N | Y by16 | N | | AES256-CCM | N | Y by8 | Y by8 | N | N | Y by16 | N | | AES128-CBC | N | Y(1) | Y(3) | N | N | Y(6) | N | @@ -129,9 +143,9 @@ Table 2. List of supported integrity algorithms and their implementations. | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | N | | SHA2-384 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | N | | SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | N | -| AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | -| AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | -| AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | +| AES128-GMAC | N | Y by8 | N | Y by8 | Y by8 | Y by32 | N | +| AES192-GMAC | N | Y by8 | N | Y by8 | Y by8 | Y by32 | N | +| AES256-GMAC | N | Y by8 | N | Y by8 | Y by8 | Y by32 | N | | NULL | Y | N | N | N | N | N | N | | AES128-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 | N | | AES256-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 | N | @@ -143,10 +157,10 @@ Table 2. List of supported integrity algorithms and their implementations. | SNOW3G-UIA2(8) | N | Y by4 | Y by4 | N | Y by32 | Y by32 | Y | | DOCSIS-CRC32(4) | N | Y | Y | N | Y | Y | N | | HEC | N | Y | Y | N | N | N | N | -| POLY1305 | Y | N | N | N | Y | Y | N | -| POLY1305 AEAD | Y | N | N | N | Y | Y | N | -| SNOW-V AEAD | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | -| GHASH | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | N | +| POLY1305 | Y | N | N | Y(9) | Y | Y | N | +| POLY1305 AEAD | Y | N | N | Y(9) | Y | Y | N | +| SNOW-V AEAD | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by32 | N | +| GHASH | N | Y by8 | N | Y by8 | Y by8 | Y by32 | N | | CRC(6) | N | Y by8 | Y by8 | N | N | Y by16 | N | | PON-CRC-BIP(7) | N | Y | Y | N | N | Y | N | +-----------------------------------------------------------------------------------+ @@ -168,6 +182,7 @@ Notes: - CRC6: IUUP header (7) - used only with PON-AES128-CTR cipher (8) - x4/x16 for init keystream generation, then by4/by32 +(9) - Only if AVX-IFMA instructions are supported Legend: ` byY`- single buffer Y blocks at a time @@ -221,8 +236,7 @@ Table 3. Encryption and integrity algorithm combinations +---------------+-----------------------------------------------------+ ``` -2\. Processor Extensions -======================== +## 2. Processor Extensions Table 4. Processor extensions used in the library ``` @@ -246,8 +260,7 @@ Table 4. Processor extensions used in the library +-------------------+-----------+-----------------------------------------+ ``` -3\. Recommendations -=================== +## 3. Recommendations Legacy or to be avoided algorithms listed in the table below are implemented in the library in order to support legacy applications. Please use corresponding @@ -275,8 +288,7 @@ Operating System to provide process isolation. As the result, it is recommended to use latest Operating System patches and security updates. -4\. Package Content -=================== +## 4. Package Content - test - Library test applications - perf - Library performance application @@ -290,20 +302,18 @@ security updates. **Note:** There is just one branch used in the project. All development is done on the main branch. +Code taken from the tip of the main branch should not be considered fit for production. -5\. Documentation -=============== +## 5. Documentation Full documentation can be found at: https://intel.github.io/intel-ipsec-mb To generate documentation locally, run: `> make doxy` -6\. Compilation -=============== +## 6. Compilation -Linux (64-bit only) -------------------- +### Linux (64-bit only) Required tools: - GNU make @@ -332,15 +342,14 @@ Build with AESNI emulation support (disabled by default): For more build options and their explanation run: `> make help` -Windows MSVS (x64 only) ------------------- +### Windows MSVS (x64 only) Required tools: -- Microsoft (R) Visual Studio 2015: - - NMAKE: Microsoft (R) Program Maintenance Utility Version 14.00.24210.0 - - CL: Microsoft (R) C/C++ Optimizing Compiler Version 19.00.24215.1 for x64 - - LIB: Microsoft (R) Library Manager Version 14.00.24215.1 - - LINK: Microsoft (R) Incremental Linker Version 14.00.24215.1 +- Microsoft (R) Visual Studio 2019: + - NMAKE: Microsoft (R) Program Maintenance Utility Version 14.29.30148.0 + - CL: Microsoft (R) C/C++ Optimizing Compiler Version 19.29.30148 for x64 + - LIB: Microsoft (R) Library Manager Version 14.29.30148.0 + - LINK: Microsoft (R) Incremental Linker Version 14.29.30148.0 - Note: Building on later versions should work but is not verified - NASM version 2.14 (or newer) @@ -376,8 +385,7 @@ Build with AESNI emulation support (disabled by default): For more build options and their explanation run: `> nmake /f win_x64.mak help` -Windows Mingw-w64 (64-bit only) -------------------- +### Windows Mingw-w64 (64-bit only) Required tools: - GNU mingw32-make.exe @@ -403,8 +411,7 @@ Build with debugging information: For more build options and their explanation run: `> mingw32-make.exe help` -FreeBSD (64-bit only) ---------------------- +### FreeBSD (64-bit only) Required tools: - GNU make @@ -430,8 +437,81 @@ Build with debugging information: For more build options and their explanation run: `> gmake help` -7\. Security Considerations & Options for Increased Security -============================================================ +### Building with CMake (experimental) + +Minimum CMake version: 3.16 + +Create build directory: +``` +mkdir build +cd build +``` + +#### Unix Makefiles (Linux and FreeBSD) + +Shared library (default): +``` +cmake .. +cmake --build . --parallel +``` + +Static library: +``` +cmake -DBUILD_SHARED_LIBS=OFF .. +cmake --build . --parallel +``` + +Debug build: +``` +cmake -DCMAKE_BUILD_TYPE=Debug .. +cmake --build . --parallel +``` + +For more build options and their explanation run: +`cmake --build . --target print_help` + +#### Windows MSVS (x64 only) + +Shared library with debugging information (default for MSVS) +``` +cmake -Ax64 .. +cmake --build . +``` + +Release build: +``` +cmake -Ax64 .. +cmake --build . --config Release +``` + +Static library: +``` +cmake -Ax64 -DBUILD_SHARED_LIBS=OFF .. +cmake --build . --config Release +``` + +For more build options and their explanation run: +`cmake --build . --target print_help` + +#### Ninja (Linux, FreeBSD and Windows): +``` +cmake -G Ninja .. +cmake --build . +``` + +For more build options and their explanation run: +``` +cmake --build . --target print_help +``` + +Library and applications can be found in: +``` +build/lib +build/test +build/perf +``` + +## 7. Security Considerations & Options for Increased Security ### Security Considerations The security of a system that uses cryptography depends on the strength of @@ -467,6 +547,7 @@ options are listed below as the "Options for Increased Security" and are enabled through various build flags. ### Options for Increased Security + There are three build options that are used to increase safety in the code and help protect external functions from incorrect input data. The SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP options are enabled by default. @@ -478,14 +559,17 @@ tests have been performed to help protect against or check for side-channel attacks. ### SAFE_DATA + Stack and registers containing sensitive information, such as keys or IVs, are cleared upon completion of a function call. ### SAFE_PARAM + Input parameters are checked, looking generally for NULL pointers or an incorrect input length. ### SAFE_LOOKUP + Lookups which depend on sensitive information are implemented with constant time functions. @@ -500,12 +584,14 @@ algorithms listed above may be susceptible to timing attacks which could expose the cryptographic key. ### SAFE_OPTIONS + SAFE_OPTIONS is a parameter that can be used to disable/enable all supported safe options (i.e. SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). It is set to `y` by default and all safe options are enabled. `SAFE_OPTIONS=n` disables all safe options. ### Security API + **Force clearing/zeroing of memory** ```c IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size); @@ -516,15 +602,14 @@ of memory pointed to by _'mem'_ followed by the _sfence_ instruction to ensure memory is cleared before the function returns. ### Galois Counter Mode (GCM) TAG Size + The library GCM and GMAC implementation provides flexibility as to tag size selection. As explained in [NIST Special Publication 800-38D](https://csrc.nist.gov/publications/detail/sp/800-38d/final) section 5.2.1.2 and Appendix C, using tag sizes shorter than 96 bits can be insecure. Please refer to the aforementioned sections to understand the details, trade offs and mitigations of using shorter tag sizes. -8\. Installation -================ +## 8. Installation -Linux (64-bit only) -------------------- +### Linux (64-bit only) First compile the library and then install: `> make` @@ -542,8 +627,7 @@ If there is no need to run ldconfig at install stage please use NOLDCONFIG=y opt If library was compiled as an archive (not a default option) then install it using SHARED=n option: `> sudo make install SHARED=n` -Windows (x64 only) ------------------- +### Windows (x64 only) First compile the library and then install from a command prompt in administrator mode: `> nmake /f win_x64.mak` @@ -558,8 +642,7 @@ If you want to change install location then define PREFIX (default C:\Program Fi If library was compiled as a static library (not a default option) then install it using SHARED=n option: `> nmake /f win_x64.mak install SHARED=n` -FreeBSD (64-bit only) -------------------- +### FreeBSD (64-bit only) First compile the library and then install: `> gmake` @@ -577,24 +660,48 @@ If there is no need to run ldconfig at install stage please use NOLDCONFIG=y opt If library was compiled as an archive (not a default option) then install it using SHARED=n option: `> sudo gmake install SHARED=n` -9\. Backwards compatibility -=========================== +## Installing with CMake (experimental) + +### Unix (Linux and FreeBSD) + +First compile the library and then install: +``` +cmake --build . +sudo cmake --install . +``` + +To uninstall the library run: +`sudo cmake --build . --target uninstall` + +If you want to change install location then define PREFIX: +`sudo cmake --install . --prefix=` + +### Windows (x64 only) + +First compile the library and then install from a command prompt in administrator mode: +``` +cmake --build . --config Release +cmake --install . --config Release +``` + +To uninstall the library run: +`cmake --build . --target uninstall` + +If you want to change install location then define PREFIX (default C:\Program Files): +`cmake --install . --config Release --prefix=` -In version 0.54, some symbols have been renamed to avoid too generic names (such as cipher modes or -hash algorithms). +## 9. Backwards compatibility -When building an application and linking it against the IPSec Multi Buffer library, -by default the old symbols (up to v0.53) are exported, to maintain backwards compatibility. +In version 1.4, backward compile time symbol compatibility with +library version 0.53 has been removed. -Applications should move to the new API as soon as possible, as the old symbols -are marked as deprecated and will be removed in a future release. +Applications are encouraged to use new symbol names introduced in version 0.54. -For applications which face symbol conflicts due to these old generic names, -they should be compiled with the flag -DNO_COMPAT_IMB_API_053, which will -not export the old symbols. +If required, compatibility symbol mapping can be implemented in the application. +See compatibility symbol mapping in v1.3 header file: +https://github.com/intel/intel-ipsec-mb/blob/v1.3/lib/intel-ipsec-mb.h#L246 -10\. Disclaimer (ZUC, KASUMI, SNOW3G) -==================================== +## 10. Disclaimer (ZUC, KASUMI, SNOW3G) Please note that cryptographic material, such as ciphering algorithms, may be subject to national regulations. What is more, use of some algorithms in @@ -604,3 +711,185 @@ licensing by the GSMA and/or the ETSI. For more details please see: - GSMA https://www.gsma.com/security/security-algorithms/ - ETSI https://www.etsi.org/security-algorithms-and-codes/cellular-algorithm-licences + +## 11. Legal Disclaimer + +THIS SOFTWARE IS PROVIDED BY INTEL"AS IS". NO LICENSE, EXPRESS OR +IMPLIED, BY ESTOPPEL OR OTHERWISE, TO ANY INTELLECTUAL PROPERTY RIGHTS +ARE GRANTED THROUGH USE. EXCEPT AS PROVIDED IN INTEL'S TERMS AND +CONDITIONS OF SALE, INTEL ASSUMES NO LIABILITY WHATSOEVER AND INTEL +DISCLAIMS ANY EXPRESS OR IMPLIED WARRANTY, RELATING TO SALE AND/OR +USE OF INTEL PRODUCTS INCLUDING LIABILITY OR WARRANTIES RELATING TO +FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR INFRINGEMENT +OF ANY PATENT, COPYRIGHT OR OTHER INTELLECTUAL PROPERTY RIGHT. + +## 12. FIPS Compliance + +The library does not fulfill technical requirements to achieve Cryptographic Module (CMVP) certification as a standalone component. It is fit for Cryptographic Algorithm validation and certification (CAVP) and it can be part of CMVP as one of the components. + +### CAVP + +ACVP test application located in `test` directory is to support CAVP process. It implements validation of the following algorithms: +- AES-GCM +- AES-GMAC +- AES-CCM +- AES-CBC +- AES-EBC +- TDES-EDE-CBC +- AES-CTR +- AES-CMAC +- SHA1 (SHA-1) +- SHA224 (SHA2-224) +- SHA256 (SHA2-256) +- SHA384 (SHA2-384) +- SHA512 (SHA2-512) +- HMAC-SHA1 (HMAC-SHA-1) +- HMAC-SHA224 (HMAC-SHA2-224) +- HMAC-SHA256 (HMAC-SHA2-256) +- HMAC-SHA384 (HMAC-SHA2-384) +- HMAC-SHA512 (HMAC-SHA2-512) + +### CAVP Algorithm Parameters + +**Note:** all sizes in bits +``` ++--------------------------------------------------------------------------------------------+ +| Algorithm | Standard | Parameters | +|---------------------+-----------+----------------------------------------------------------| +| AES-GCM | SP800-38D | Key size: 128, 192, 256 | +| | | Direction: encrypt and decrypt | +| | | ivLen: [min = 8, max = 1024, increment 8] | +| | | tagLen: 32, 64, 96, 104, 112, 120, 128 | +| | | payloadLen: [min = 0, max = 65536, increment = 8] | +| | | aadLen: [min = 0, max = 65536, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-CBC | SP800-38A | Key size: 128, 192, 256 | +| | | Direction: encrypt and decrypt | +|---------------------+-----------+----------------------------------------------------------| +| AES-CTR | SP800-38A | Key size: 128, 192, 256 | +| | | Direction: encrypt and decrypt | +| | | payloadLen: [min = 8, max = 128, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-ECB | SP800-38A | Key size: 128, 192, 256 | +| | | Direction: encrypt and decrypt | +|---------------------+-----------+----------------------------------------------------------| +| TDES-EDE-CBC | SP800-38A | Key size: 192 | +| | | Direction: encrypt and decrypt | +|---------------------+-----------+----------------------------------------------------------| +| SHA1 (SHA-1) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA224 (SHA2-224) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA256 (SHA2-256) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA384 (SHA2-384) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA512 (SHA2-512) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-CCM | SP800-38C | Key size: 128, 256 | +| | | Direction: encrypt and decrypt | +| | | ivLen: [min = 56, max = 104, increment 8] | +| | | tagLen: 32, 48, 64, 80, 96, 112, 128 | +| | | payloadLen: [min = 0, max = 256, increment = 8] | +| | | aadLen: [min = 0, max = 368, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-GMAC | SP800-38B | Key size: 128, 192, 256 | +| | | ivLen: [min = 8, max = 1024, increment 8] | +| | | tagLen: 32, 64, 96, 104, 112, 120, 128 | +| | | aadLen: [min = 0, max = 65536, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-CMAC | SP800-38B | Key size: 128, 256 | +| | | msgLen: [min = 8, max = 65528, increment 8] | +| | | macLen: [min = 8, max = 128, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA1 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA-1) | | macLen: [min = 32, max = 160, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA224 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-224) | | macLen: [min = 32, max = 224, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA256 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-256) | | macLen: [min = 32, max = 256, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA384 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-384) | | macLen: [min = 32, max = 384, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA512 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-512) | | macLen: [min = 32, max = 512, increment = 8] | ++--------------------------------------------------------------------------------------------+ +``` +### Self-Test + +In order to support CMVP, the library implements Self-Test functionality that is available with all compilation options. +The test is always performed as part of library initialization (power-up). There is no conditional self-test functionality as none of such conditions occur (i.e. pair-wise consistency test, +software/firmware load test, manual key entry test, continuous random number generator test, and +bypass test). +The self-test consists of Cryptographic algorithm test (known answer test) on following algorithms: +- AES-GCM +- AES-GMAC +- AES-CCM +- AES-CBC +- AES-CTR +- AES-ECB +- TDES-EDE-CBC +- AES-CMAC +- SHA1 +- SHA224 +- SHA256 +- SHA384 +- SHA512 +- HMAC-SHA1 +- HMAC-SHA224 +- HMAC-SHA256 +- HMAC-SHA384 +- HMAC-SHA512 + +Example detection of library self-test completion & error in the application: +``` +IMB_ARCH arch; +IMB_MGR *p_mgr = alloc_mb_mgr(0); + +init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ + +/* + * check for self-test presence and successful + * - requires library version v1.3 or newer + */ +if (p_mgr->features & IMB_FEATURE_SELF_TEST) { + /* self-test feature present */ + if (p_mgr->features & IMB_FEATURE_SELF_TEST_PASS) { + printf("SELF-TEST: PASS\n"); + } else { + printf("SELF-TEST: FAIL\n"); + } +} else { + printf("SELF-TEST: N/A (requires library >= v1.3)\n"); +} + +/* check for initialization self-test error */ +if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { + /* self-test error */ + exit(EXIT_FAILURE); +} +``` + +## 13.DLL Injection Attack + +### Problem + +The Windows OS has an insecure predefined search order and set of defaults when trying to locate a resource. If the resource location is not specified by the software, an attacker need only place a malicious version in one of the locations Windows will search, and it will be loaded instead. Although this weakness can occur with any resource, it is especially common with DLL files. + +### Solutions + +Applications using intel-ipsec-mb DLL library may need to apply one of the solutions to prevent from DLL injection attack. + +Two solutions are available: +- Using a Fully Qualified Path is the most secure way to load a DLL +- Signature verification of the DLL + +### Resources and Solution Details + +- Security remarks section of LoadLibraryEx documentation by Microsoft: +- Microsoft Dynamic Link Library Security article: +- Hijack Execution Flow: DLL Search Order Hijacking: +- Hijack Execution Flow: DLL Side-Loading: diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt new file mode 100644 index 0000000000000000000000000000000000000000..7350d2a2645acda43466ac8a495d39ad8c31609f --- /dev/null +++ b/ReleaseNotes.txt @@ -0,0 +1,875 @@ +======================================================================== +Release Notes for Intel(R) Multi-Buffer Crypto for IPsec Library + +v1.4 June 2023 +======================================================================== + +General +- Experimental CMake support for Linux, FreeBSD and Windows added + +Library +- POLY1305 AVX2 with AVX-IFMA instructions added. +- Optimized GHASH component in AVX512 VAES (type2) AES-GCM implementation. +- Implemented a workaround for false load-block condition in SSE and AVX2 AES-GCM implementations. +- Removed AVX AES-GCM implementation, its API symbols map to the SSE implementation. +- QUIC header protection API added. +- QUIC AES-GCM-128/256 AEAD API added. +- Removed v0.53 (and older) compatibility symbol mapping (NO_COMPAT_IMB_API_053 not defined). +- ZUC AVX2-GFNI implementation added. +- SHA-NI instructions enabled to use in SHA1/224/256 direct API +- New API (imb_set_session) added to be used with burst API, helping speeding up the crypto scheduling. +- New API added to calculate IPAD/OPAD for SHAx-HMAC. +- New direct API added to calculate DES-CFB and AES-CFB-256 on a single block. + +Test Applications +- ACVP test application extended to support: AES-ECB and 3DES-CBC. +- Added sample applications showcasing how to use the new burst API. +- CMake support added, including ability to run tests with it. +- Extended fuzzing app to cover remaining direct APIs. +- Test vector standardized for various algorithms (SHA/XCBC/POLY1305/CMAC/GMAC/GHASH/HMAC-SHAx/MD5). +- Changed `test` directory structure and test application names. Each test application has its own subdirectory. + +Performance Application +- New parameter added to benchmark crypto on unaligned buffers +- Renamed performance application to `imb-perf` + +Fixes +- Fixed MB_MGR initialization corruption (issue #115) +- Fixed performance scaling issue resulting from the misuse of global `errno` variable (issue #112) + +v1.3 September 2022 +======================================================================== + +Library +- ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE, AVX, AVX2 and AVX512 +- AES-ECB AVX512-VAES implementation added +- AES-ECB optimizations for AVX and SSE +- AES-ECB AVX2-VAES implementation added +- JOB API GHASH support added +- SHA1/224/256/384/512 multi-buffer implementation added +- Multi-buffer SHA1, SHA224 and SHA256 use SHANI if available +- Synchronous cipher and hash burst API added + - cipher API only supports AES-CBC and AES-CTR + - hash API only supports HMAC-SHA1, HMAC-224, HMAC-256, HMAC-384 and HMAC-512 +- Asynchronous burst API added that supports all cipher and hash modes +- SNOW3G-UEA2 SSE multi-buffer implementation added +- SNOW3G-UIA2 SSE multi-buffer initialization and key-stream generation added +- SNOW3G-UEA2 and SNOW3G-UIA2 SSE implementation used in JOB API for + AVX and AVX2 architectures +- API documentation added (doxygen generated) +- New SGL job API (AES-GCM and CHACHA20-POLY1305 only) +- Enforced EVEX PMADD52 encoding in AVX512 code +- Restructured reset flow of architecture managers +- SSE, AVX, AVX2 and AVX512 managers were split to better cover different types +- Added library self-test functionality +- enbranch64 not emitted on Windows builds (CET related) +- use SHANI extensions in AVX2 type-2 and AVX type-2 for SHA224, HMAC-SHA224, + SHA256 and HMAC-SHA256 +- use SHANI extensions in AVX type-2 for SHA1, HMAC-SHA1 +- no-GFNI option added to help with testing +- single buffer SHANI implementation of SHA1 and SHA256 added +- single buffer SHANI implementation used in HMAC-SHA1, HMAC-SHA224 and + HMAC-SHA256 flush operation + +Test Applications +- GHASH JOB API support added in the test application, fuzzing and xvalid tools +- Burst API support added for supported algorithms +- ACVP test application extended to support: AES-GCM, AES-GMAC, AES-CCM, + AES-CBC, AES-CTR, AES-CMAC, SHA1, SHA224, SHA256, SHA384, SHA512, HMAC-SHA1, + HMAC-SHA224, HMAC-SHA256, HMAC-SHA384, HMAC-SHA512 +- Cross validation (xvalid) tool improvements in pattern search functionality +- FreeBSD added to github CI +- Added AVX-SSE transition check to the cross validation tool (xvalid) +- Wycheproof AES-GCM, AES-CCM, CHACHA20-POLY1305, AES-CMAC, AES-GMAC, HMAC-SHA1, + HMAC-SHA224, HMAC-SHA256, HMAC-SHA384 and HMAC-SHA512 test vectors added + to a new test tool +- no-GFNI option added +- Fuzzing application extended to cover new burst API's + +Performance Application +- GHASH support added (through JOB and direct API) +- CHACHA20-POLY1305 support through direct API +- Support added for SHA1/224/256/384/512 +- Burst API support added for supported algorithms +- SGL support added (AES-GCM and CHACHA20-POLY1305 only) +- no-GFNI option added + +Fixes +- Fixed 23-byte IV expansion for ZUC-256 (issue #102) +- Fixed incorrect 8-buffer SNOW3G key-stream generation (issue #104) +- Numerous AVX-SSE transition fixes with SAFE_OPTIONS=n +- [ZUC-EIA3] allow unaligned digest load/stores +- AES-CCM authentication flush may load out of scope data (issue #107) +- AES-CMAC authentication flush may load out of scope data (similar to issue #107) + +v1.2 February 2022 +======================================================================== + +General +- Windows CET support +- Disabled AESNI emulation support by default and make it optional + +Library +- Generation of PDB in release build on Windows added +- SAFE_OPTIONS option added to unify SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP options + +Test Applications +- Extended invalid IV length tests +- Test application improvements added +- Fuzz testing tool improvements added +- Auto-generation of direct API invalid parameters tests added +- ACVP test application added + +Performance Application +- GCM support for SGL API added + +Fixes +- Fixed incorrect job length calculation in CBCS encryption +- Fixed FreeBSD build (intel/intel-ipsec-mb#94) +- Added missing checks for HMAC IPAD and opad +- Added missing checks for XCBC K1, K2 and K3 + +v1.1 October 2021 +======================================================================== + +General +- Added support to build with Mingw-w64 on Windows + +Library +- PON algorithm AVX512-VAES implementation added +- SNOW3G-UIA2 AVX512 and AVX512-VPCLMULQDQ implementations added +- SNOW3G-UEA2 AVX512 and AVX512-VAES implementations added +- SNOW-V AVX implementation added +- ZUC optimizations for AVX512 +- GCM optimizations for AVX512 +- Poly1305 optimizations for AVX512-VAES +- Improved error code handling (missing mainly on assembly modules) +- ZUC-256 23-byte IV support added + +Test Applications +- Error handling tests added (for job and direct API) +- Fuzz testing added + +v1.0 April 2021 +======================================================================== + +General +- Top level `lib` directory tidy up + - build scripts and header file left at the top level + - `lib/x86_64` directory created + - files requiring compilation moved from `lib/include` +- Symbols not stripped from static library at installation +- API name changes and unification + - mapping provided for backwards compatibility +- NASM version check in the build script +- CET enabling in the build scripts + +Library +- CET enabling (endbranch opcodes added) +- ZUC-EIA3-256 support for SSE, AVX, AVX2 and AVX512 (VAES) + - 4 byte tag length only +- Chacha20 optimizations for SSE, AVX and AVX2 +- ZUC-EEA3-256 support for SSE, AVX, AVX2 and AVX512 (VAES) +- SNOW-V and SNOW-V-AEAD support for SSE +- Poly1305 AVX512 and AVX512-IFMA implementations added +- Chacha20-Poly1305 AEAD implementations extended to AVX512 and AVX512-IFMA +- CBCS AVX512 optimizations +- Extended CBCS to return last cipher block to maintain context between calls +- AVX/SSE transition fixes +- Added SGL support for AEAD Chacha20-Poly1305 +- Poly1305 minor optimization in the scalar code +- GHASH API change +- IFMA CPU feature detection +- SGL support added for AES-GCM through job API +- Added CRC functions through job API + +Test Applications +- ZUC-EEA3-256 tests added to test and xvalidation applications +- SNOW-V and SNOW-V-AEAD tests added to test and xvalidation applications +- IMIX support added to the xvalidation application +- AEAD Chacha20-Poly1305 tests added +- SGL tests added for AES-GCM through job API +- CRC function tests added through job API + +Performance Application +- ZUC-EEA3-256 support added +- SNOW-V and SNOW-V-AEAD support added +- AEAD Chacha20-Poly1305 support added +- Created `ipsec_perf_tool.py` to run multiple `ipsec_perf` + instances at the same time +- DOCSIS cipher combined with CRC32 treated as AEAD algorithm +- CRC functions added + +v0.55 October 2020 +======================================================================== + +General +- Restructured project to move all library code into new 'lib' directory +- Renamed LibPerfApp directory to perf +- Renamed LibTestApp directory to test + +Library +- AES-CCM-256 implementation for SSE, AVX and AVX512 (VAES) +- AES-CMAC-256 implementation for SSE, AVX and AVX512 (VAES) +- 32bit and 64bit HEC compute API added +- AES-GMAC direct API added to support Scatter-Gather list (SGL) +- CALC_AAD_HASH macro improved for AVX512 (VAES), boosting performance + for AES-GMAC, GHASH and hash calculation for AAD in AES-GCM +- ZUC-EEA3 and ZUC-EIA3 Multi-buffer implemented for SSE using + GFNI instructions. +- AES-XCBC-128 implementation for AVX512 (VAES) +- AES-CBCS-128 implementation for SSE and AVX (1:9 crypt:skip pattern) +- Chacha20 SSE, AVX and AVX512 implementations +- Automatic multi-buffer manager initialization API added +- Error handling API added +- Build with SAFE_DATA and SAFE_PARAM options by default +- Poly1305 scalar implementation +- AEAD Chacha20-Poly1305 implementation +- CRC implementation for RNC, LTE, WiMAX, SCTP, Ethernet and CRC16 CCIT + +Test Applications +- CCM tests extended to test AES-CCM-256 +- CMAC tests extended to test AES-CMAC-256 +- HEC tests added to test app +- AES-GMAC SGL tests added to test app +- AES-XCBC-128 tests added to test app +- AES-CBCS-128 tests added to test app +- AES-CBCS-128 support added to xvalidation app +- Chacha20 tests added to test and xvalidation app +- Poly1305 tests added to test and xvalidation app +- AEAD Chacha20-Poly1305 tests added to test and xvalidation app +- CRC tests added +- Automatic architecture detection done by default + +Performance Application +- AES-CCM-256 support added +- AES-CMAC-256 support added +- AES-CBCS-128 support added +- Chacha20 support added +- Poly1305 support added +- AEAD Chacha20-Poly1305 support added + + +v0.54 April 2020 +======================================================================== + +Library +- ZUC-EEA3 and ZUC-EIA3 algorithms added in job API (using cipher mode + IMB_CIPHER_ZUC_EEA3 and hash_alg IMB_AUTH_ZUC_EIA3_BITLEN) +- SNOW3G-UEA2 and SNOW3G-UIA2 algorithms added in job API (using cipher type + IMB_CIPHER_SNOW3G_UEA2_BITLEN and hash type IMB_AUTH_SNOW3G_UIA2_BITLEN) +- AVX512 implementation of stitched DOCSIS cipher with CRC32 calculations +- KASUMI-UEA1 and KASUMI-UIA1 algorithms added in job API (using cipher type + IMB_CIPHER_KASUMI_UEA1_BITLEN and hash type IMB_AUTH_KASUMI_UIA1) +- New GHASH API added +- ZUC-EIA3 Multi-buffer API added and implemented for SSE and AVX. +- Added support for any IV size in AES-GCM, through the job API and new + direct API +- Check for new flag NO_COMPAT_IMB_API_053, which exposes only new API, + removing backwards compatibility with version v0.53. +- AES-CMAC implementation for VAES added +- SSE AES128-CTR, AES192-CTR, AES256-CTR and AES128-CCM by8 cipher + implementations added +- Added by8/x8 implementations of SSE AES128-CBC, AES192-CBC, AES256-CBC, + DOCSIS SEC BPI, AES-CCM and AES-CMAC. +- AES256-DOCSIS algorithm added. +- ZUC-EEA3 and ZUC-EIA3 Multi-buffer implemented for AVX2 and AVX512. +- ZUC-EEA3 and ZUC-EIA3 Multi-buffer implemented for AVX2 and AVX512, + using this latter GFNI and VAES instructions where these are present + in the CPU. +- Minimum required version for NASM is now 2.14. +- ZUC-EEA3 and ZUC-EIA3 Multi-buffer implemented with AESNI emulation instructions. +- SNOW3G-UIA2 and SNOW3G-UEA2 reimplemented for increased security and performance. +- AES-CBC improvement for VAES +- AES-CCM implementation for VAES added + +LibTestApp +- Extended ZUC tests to validate ZUC-EEA3 and ZUC-EIA3 algorithms through + job API +- Extended SNOW3G tests to validate SNOW3G-UEA2 and SNOW3G-UIA2 algorithms + through job API +- Extended DOCSIS tests with combined CRC32 calculation cases +- Extended KASUMI tests to validate KASUMI-UEA1 and KASUMI-UIA1 algorithms + through job API +- Extended ZUC tests to validate ZUC-EIA3 multi-buffer implementation + through direct and job API +- Extended AES-DOCSIS tests with 256-bit keys + +LibPerfApp +- Added support for ZUC-EEA3 and ZUC-EIA3 algorithms +- Added support for SNOW3G-UEA2 and SNOW3G-UIA2 algorithms +- Added support for DOCSIS combined with CRC32 +- Added support for KASUMI-UEA1 and KASUMI-UIA1 algorithms + +v0.53 October 2019 +======================================================================== + +Library +- AES-CCM performance optimizations done + - full assembly implementation + - authentication decoupled from cipher + - CCM chain order expected to be HASH_CIPHER for encryption and + CIPHER_HASH for decryption +- AES-CTR implementation for VAES added +- AES-CBC implementation for VAES added +- Single buffer AES-GCM performance improvements added for VPCLMULQDQ + VAES +- Multi-buffer AES-GCM implementation added for VPCLMULQDQ + VAES +- Data transposition optimizations and unification across the library + implemented +- Generation of make dependency files for Linux added +- AES-ECB implementation added +- PON specific stitched algorithm implementation added + - stitched AES-CTR-128 (optional) with CRC32 and BIP (running 32-bit XOR) +- AES-CMAC-128 implementation for bit length messages added +- ZUC-EEA3 and ZUC-EIA3 implementation added +- FreeBSD experimental support added +- KASUMI-F8 and KASUMI-F9 implementation added +- SNOW3G-UEA2 and SNOW3G-UIA2 implementation added +- AES-CTR implementation for bit length (128-NEA2/192-NEA2/256-NEA2) messages added +- SAFE_PARAM, SAFE_DATA and SAFE_LOOKUP compile time options added. + Find more about these options in the README file or on-line at + https://github.com/intel/intel-ipsec-mb/blob/master/README. + +LibTestApp +- New API tests added +- CMAC test vectors extended +- New chained operation tests added +- Out-of-place chained operation tests added +- AES-ECB tests added +- PON algorithm tests added +- Extra AES-CTR test vectors added +- Extra AES-CBC test vectors added +- AES-CMAC-128 bit length message tests added +- CPU capability detection used to disable tests if instruction not present +- ZUC-EEA3 and ZUC-EIA3 tests added +- New cross architecture test application (ipsec_xvalid) added, + which mixes different implementations (based on different architectures), + to double check their correctness +- SNOW3G-UEA2 and SNOW3G-UIA2 tests added +- AES-CTR-128 bit length message tests added +- Negative tests extended to cover all API's + +LibPerfApp +- Job size and number of iterations options added +- Single architecture test option added +- AAD size option added +- Allow zero length source buffer option added +- Custom performance test combination added: + cipher-algo, hash-algo and aead-algo arguments. +- Cipher direction option added +- The maximum buffer size extended from 2K to 16K +- Support for user defined range of job sizes added + +Fixes +- Uninitialized memory reported by Valgrind fixed +- Flush decryption job fixed (issue #33) +- NULL_CIPHER order check removed (issue #30) +- Save XMM registers when emulating AES fixed (issue #28) +- SSE & AVX AES-CMAC fixed (issue #27) +- Missing GCM pointers fixed for AES-NI emulation (issue #29) + +v0.52 December 2018 +======================================================================== + +03 Dec, 2018 + +General +- Added AESNI emulation implementation +- Added AES-GCM multi-buffer implementation for AVX512 +- Added flexible job chain order support +- GCM submit and flush functions moved into architecture MB manager modules +- AVX512/AVX2/AVX/SSE AAD GHASH computation performance improvement +- GCM API's added to MB_MGR structure +- Added plain SHA support in JOB API +- Added architectural compiler optimizations for GCC/CC + +LibTestApp +- Added option not to run GCM tests +- Added AESNI emulation tests +- Added plain SHA tests +- Updated to take advantage of new GCM macros + +LibPerfApp +- Buffer alignment update +- Updated to take advantage of new GCM macros + +v0.51 September 2018 +======================================================================== + +13 Sep, 2018 + +General +- AES-CMAC performance optimizations +- Implemented store to load optimizations in + - AES-CMAC submit and flush jobs for SSE and AVX + - HMAC-MD5, HMAC-SHA submit jobs for AVX + - HMAC-MD5 submit job for AVX2 +- Added zero-sized message support in GCM +- Stack execution flag disabled in new asm modules + +LibTestApp +- Added AES vectors +- Added DOCSIS AES vectors +- Added CFB validation + +LibPerfApp +- Smoke test option added + +v0.50 June 2018 +======================================================================== + +13 Jun, 2018 + +General +- Added support for compile time and runtime library version checking +- Added support for full MD5 digest size +- Replaced defines for API with symbols for binary compatibility +- Added HMAC-SHA & HMAC-MD5 vectors to LibTestApp +- Added support for zero cipher length in AES-CCM +- Added new API's to compute SHA1, SHA224, SHA256, SHA384 and SHA512 hashes + to support key reduction cases where key is longer than a block size +- Extended support for HMAC full digest sizes for HMAC-SHA1, HMAC-SHA224, + HMAC-SHA256, HMAC-SHA384 and HMAC-SHA512. Previously only truncated sizes + were supported. +- Added AES-CMAC support for output digest size between 4 and 16 bytes +- Added GHASH support for output digest size up to 16 bytes +- Optimized submit job API's with store to load optimization in SSE, AVX, + AVX2 (excluding MD5) +- Improved performance application accuracy by increase number of + test iterations +- Extended multi-thread features of LibPerfApp Windows version to match + Linux version of the application + +v0.49 March 2018 +======================================================================== + +21 Mar, 2018 + +General +- AES-CMAC support added (AES-CMAC-128 and AES-CMAC-96) +- 3DES support added +- Library compiles to SO/DLL by default +- Install/uninstall targets added to makefiles +- Multiple API header files consolidated into one (intel-ipsec-mb.h) +- Unhalted cycles support added to LibPerfApp (Linux at the moment) +- ELF stack execute protection added for assembly files +- VZEROUPPER instruction issued after AVX2/AVX512 code to avoid + expensive SSE<->AVX transitions +- MAN page added +- README documentation extensions and updates +- AVX512 DES performance smoothed out +- Multi-buffer manager instance allocate and free API's added +- Core affinity support added in LibPerfApp + +v0.48 December 2017 +======================================================================== + +12 Dec, 2017 + +General +- Linux SO compilation option added +- Windows DLL compilation option added +- AES CCM 128 support added +- Multithread command line option added to LibPerfApp +- Coding style fixes +- Coding style target added to Makefile + +v0.47 October 2017 +======================================================================== + +Oct 5, 2017 + +Intel(R) AVX-512 Instructions +- DES CBC AVX512 implementation +- DOCSIS DES AVX512 implementation +General +- DES CBC cipher added (generic x86 implementation) +- DOCSIS DES cipher added (generic x86 implementation) +- DES and DOCSIS DES tests added +- RPM SPEC file created + +v0.46 June 2017 +======================================================================== + +Jun 27, 2017 + +General +- AES GCM optimizations for AVX2 +- Change of AES GCM API: renamed and expanded keys separated from the context +- New AES GCM API via job structure and API's + - use of the interface may simplify application design at the expense of + slightly lower performance vs direct AES GCM API's +- AES GCM IV automatically padded with block counter (no need for application to do it) +- IV in AES CTR mode can be 12 bytes (no block counter); 16 byte format still allowed +- Macros added to ease access to job API for specific architecture + - use of these macros can simplify application design but it may produce worse + performance than calling architecture job API's directly +- Submit_job_nocheck() API added to gain some cycles by not validating job structure +- Result stability improvements in LibPerfApp + +v0.45 March 2017 +======================================================================== + +Mar 29, 2017 + +Intel(R) AVX-512 Instructions +- Added optimized HMAC-SHA224 and HMAC-SHA256 +- Added optimized HMAC-SHA384 and HMAC-SHA512 +General +- Windows x64 compilation target +- New DOCSIS SEC BPI V3.1 cipher +- GCM128 and GCM256 updates (with new API that is scatter gather list friendly) +- GCM192 added +- Added library API benchmark tool 'ipsec_perf' and + script to compare results 'ipsec_diff_tool.py' +Bug Fixes (vs v0.44) +- AES CTR mode fix to allow message size not to be multiple of AES block size +- RSI and RDI registers clobbered when running HMAC-SHA224 or HMAC-SHA256 + on Windows using SHA extensions + +v0.44 November 2016 +======================================================================== + +Nov 21, 2016 + +Intel(R) AVX-512 Instructions +- AVX512 multi buffer manager added (uses AVX2 implementations by default) +- Optimized SHA1 implementation added +Intel(R) SHA Extensions +- SHA1, SHA224 and SHA256 implementations added for Intel(R) SSE +General +- NULL cipher added +- NULL hash added +- NASM tool chain compilation added (default) + +======================================= +Feb 11, 2015 + +Fixed, so that the job auth_tag_output_len_in_bytes takes a different +value for different MAC types. In particular, the valid values are(in bytes): +SHA1 - 12 +sha224 - 14 +SHA256 - 16 +sha384 - 24 +SHA512 - 32 +XCBC - 12 +MD5 - 12 + +======================================= +Oct 24, 2011 + +SHA_256 added to multibuffer +------------------------ +12 Aug 2011 + +API + + The GCM API is distinct from the Multi-buffer API. This is because + the GCM code is an optimized single-buffer implementation. By + packaging them separately, the application has the option of where, + when, and how to call the GCM code, independent of how it is calling + the multi-buffer code. + + For example, the application might be enqueuing multi-buffer requests + for a separate thread to process. In this scenario, if a particular + packet used GCM, then the application could choose whether to call + the GCM routines directly, or whether to enqueue those requests and + have the compute thread call the GCM routines. + +GCM API + + The GCM functions are defined as described the the header + files. They are simple computational routines, with no state + associated with them. + +Multi-Buffer API: Two Sets of Functions + + There are two parallel interfaces, one suffixed with "_sse" and one + suffixed with "_avx". These are functionally equivalent. The "_sse" + functions work on WSM and later processors. The "_avx" functions + offer better performance, but they only run on processors after WSM. + + The same interface object structures are used for both sets of + interfaces, although one cannot mix the two interfaces on the same + initialized object (e.g. it would be wrong to initialize with + init_mb_mgr_sse() and then to pass that to submit_job_avx() ). After + the MB_MGR structure has been initialized with one of the two + initialization functions (init_mb_mgr_sse() or init_mb_mgr_avx()), + only the corresponding functions should be used on it. + + There are several ways in which an application could use these + interfaces. + + 1) Direct + If an application is only going to be run on a post-WSM machine, + it can just call the "_avx" functions directly. Conversely, if it + is just going to be run on WSM machines, it can call the "_sse" + functions directly. + + 2) Via Branches + If an application can run on both WSM and SNB and wants the + improved performance on SNB, then it can use some method to + determine if it is on SNB, and then use a conditional branch to + determine which function to call. E.g. this could be wrapped in a + macro along the lines of: + #define submit_job(mb_mgr) \ + if (_use_avx) submit_job_avx(mb_mgr); \ + else submit_job_sse(mb_mgr) + + 3) Via a Function Table + One can embed the function addresses into a structure, call them + through this structure, and change the structure based on which + set of functions one wishes to use, e.g. + + struct funcs_t { + init_mb_mgr_t init_mb_mgr; + get_next_job_t get_next_job; + submit_job_t submit_job; + get_completed_job_t get_completed_job; + flush_job_t flush_job; + }; + + funcs_t funcs_sse = { + init_mb_mgr_sse, + get_next_job_sse, + submit_job_sse, + get_completed_job_sse, + flush_job_sse + }; + funcs_t funcs_avx = { + init_mb_mgr_avx, + get_next_job_avx, + submit_job_avx, + get_completed_job_avx, + flush_job_avx + }; + funcs_t *funcs = &funcs_sse; + ... + if (do_avx) + funcs = &funcs_avx; + ... + funcs->init_mb_mgr(&mb_mgr); + + For simplicity in the rest of this document, the functions will be + referred to no suffix. + +API: Overview + + The basic unit of work is a "job". It is represented by a + JOB_AES_HMAC structure. It contains all of the information needed to + perform encryption/decryption and SHA1/HMAC authentication on one + buffer for IPSec processing. + + The basic paradigm is that the application needs to be able to + provide new jobs before old jobs have completed processing. One + might call this an "asynchronous" interface. + + The basic interface is that the application "submits" a job to the + multi-buffer manager (MB_MGR), and it may receive a completed job + back, or it may receive NULL. The returned job, if there is one, + will not be the same as the submitted job, but the jobs will be + returned in the same order in which they are submitted. + + Since there can be a semi-arbitrary number of outstanding jobs, + management of the job object is handled by the MB_MGR. The + application gets a pointer to a new job object by calling + get_next_job(). It then fills in the data fields and submits it by + calling submit_job(). If a job is returned, then that job has been + completed, and the application should do whatever it needs to do in + order to further process that buffer. + + The job object is not explicitly returned to the MB_MGR. Rather it + is implicitly returned by the next call to get_next_job(). Another + way to put this is that the data within the job object is + guaranteed to be valid until the next call to get_next_job(). + + In order to reduce latency, there is an optional function that may + be called, get_completed_job(). This returns the next job if that + job has previously been completed. But if that job has not been + completed, no processing is done, and the function returns + NULL. This may be used to reduce the number of outstanding jobs + within the MB_MGR. + + At times, it may be necessary to process the jobs currently within + the MB_MGR without providing new jobs as input. This process is + called "flushing", and it is invoked by calling flush_job(). If + there are any jobs within the MB_MGR, this will complete processing + on the earliest job and return it. It will only return NULL if there + are no jobs within the MB_MGR. + + Flushing will be described in more detail below. + + The presumption is that the same AES key will apply to a number of + buffers. For increased efficiency, it requires that the AES key + expansion happens as a distinct step apart from buffer + encryption/decryption. The expanded keys are stored in a data + structure (array), and this expanded key structure is used by the + job object. + + There are two variants provided, MB_MGR and MB_MGR2. They are + functionally equivalent. The reason that two are provided is that + they differ slightly in their implementation, and so they may have + slightly different characteristics in terms of latency and overhead. + +API: Usage Skeleton + The basic usage is illustrated in the following pseudo_code: + + init_mb_mgr(&mb_mgr); + ... + aes_keyexp_128(key, enc_exp_keys, dec_exp_keys); + ... + while (work_to_be_done) { + job = get_next_job(&mb_mgr); + // TODO: Fill in job fields + job = submit_job(&mb_mgr); + while (job) { + // TODO: Complete processing on job + job = get_completed_job(&mb_mgr); + } + } + +API: Job Fields + The mode is determined by the fields "cipher_direction" and + "chain_order". The first specifies encrypt or decrypt, and the + second specifies whether whether the hash should be done before or + after the cipher operation. + In the current implementation, only two combinations of these are + supported. For encryption, these should be set to "ENCRYPT" and + "CIPHER_HASH", and for decryption, these should be set to "DECRYPT" + and "HASH_CIPHER". + + The expanded keys are pointed to by "aes_enc_key_expanded" and + "aes_dec_key_expanded". These arrays must be aligned on a 16-byte + boundary. Only one of these is necessary (as determined by + "cipher_direction"). + + One selects AES128 vs AES256 by using the "aes_key_len_in_bytes" + field. The only valid values are 16 (AES128) and 32 (AES256). + + One selects the AES mode (CBC versus counter-mode) using + "cipher_mode". + + One selects the hash algorithm (SHA1-HMAC, AES-XCBC, or MD5-HMAC) + using "hash_alg". + + The data to be encrypted/decrypted is defined by + "src + cipher_start_src_offset_in_bytes". The length of data is + given by "msg_len_to_cipher_in_bytes". It must be a multiple of + 16 bytes. + + The destination for the cipher operation is given by "dst" (NOT by + "dst + cipher_start_src_offset_in_bytes". In many/most applications, + the destination pointer may overlap the source pointer. That is, + "dst" may be equal to "src + cipher_start_src_offset_in_bytes". + + The IV for the cipher operation is given by "iv". The + "iv_len_in_bytes" should be 16. This pointer does not need to be + aligned. + + The data to be hashed is defined by + "src + hash_start_src_offset_in_bytes". The length of data is + given by "msg_len_to_hash_in_bytes". + + The output of the hash operation is defined by + "auth_tag_output". The number of bytes written is given by + "auth_tag_output_len_in_bytes". Currently the only valid value for + this parameter is 12. + + The ipad and opad are given as the result of hashing the HMAC key + xor'ed with the appropriate value. That is, rather than passing in + the HMAC key and rehashing the initial block for every buffer, the + hashing of the initial block is done separately, and the results of + this hash are used as input in the job structure. + + Similar to the expanded AES keys, the premise here is that one HMAC + key will apply to many buffers, so we want to do that hashing once + and not for each buffer. + + The "status" reflects the status of the returned job. It should be + "STS_COMPLETED". + + The "user_data" field is ignored. It can be used to attach + application data to the job object. + +Flushing Concerns + As long as jobs are coming in at a reasonable rate, jobs should be + returned at a reasonable rate. However, if there is a lull in the + arrival of new jobs, the last few jobs that were submitted tend to + stay in the MB_MGR until new jobs arrive. This might result in there + being an unreasonable latency for these jobs. + + In this case, flush_job() should be used to complete processing on + these outstanding jobs and prevent them from having excessive + latency. + + Exactly when and how to use flush_job() is up to the application, + and is a balancing act. The processing of flush_job() is less + efficient than that of submit_job(), so calling flush_job() too + often will lower the system efficiency. Conversely, calling + flush_job() too rarely may result in some jobs seeing excessive + latency. + + There are several strategies that the application may employ for + flushing. One usage model is that there is a (thread-safe) queue + containing work items. One or more threads puts work onto this + queue, and one or more processing threads removes items from this + queue and processes them through the MB_MGR. In this usage, a simple + flushing strategy is that when the processing thread wants to do + more work, but the queue is empty, it then proceeds to flush jobs + until either the queue contains more work, or the MB_MGR no longer + contains jobs (i.e. that flush_job() returns NULL). A variation on + this is that when the work queue is empty, the processing thread + might pause for a short time to see if any new work appears, before + it starts flushing. + + In other usage models, there may be no such queue. An alternate + flushing strategy is that have a separate "flush thread" hanging + around. It wakes up periodically and checks to see if any work has + been requested since the last time it woke up. If some period of + time has gone by with no new work appearing, it would proceed to + flush the MB_MGR. + +AES Key Usage + If the AES mode is CBC, then the fields aes_enc_key_expanded or + aes_dec_key_expanded are using depending on whether the data is + being encrypted or decrypted. However, if the AES mode is CNTR + (counter mode), then only aes_enc_key_expanded is used, even for a + decrypt operation. + + The application can handle this dichotomy, or it might choose to + simply set both fields in all cases. + +Thread Safety + The MB_MGR and the associated functions ARE NOT thread safe. If + there are multiple threads that may be calling these functions + (e.g. a processing thread and a flushing thread), it is the + responsibility of the application to put in place sufficient locking + so that no two threads will make calls to the same MB_MGR object at + the same time. + +XMM Register Usage + The current implementation is designed for integration in the Linux + Kernel. All of the functions satisfy the Linux ABI with respect to + general purpose registers. However, the submit_job() and flush_job() + functions use XMM registers without saving/restoring any of them. It + is up to the application to manage the saving/restoring of XMM + registers itself. + +Auxiliary Functions + There are several auxiliary functions packed with MB_MGR. These may + be used, or the application may choose to use their own version. Two + of these, aes_keyexp_128() and aes_keyexp_256() expand AES keys into + a form that is acceptable for reference in the job structure. + + In the case of AES128, the expanded key structure should be an array + of 11 128-bit words, aligned on a 16-byte boundary. In the case of + AES256, it should be an array of 15 128-bit words, aligned on a + 16-byte boundary. + + There is also a function, sha1(), which will compute the SHA1 digest + of a single 64-byte block. It can be used to compute the ipad and + opad digests. There is a similar function, md5(), which can be used + when using MD5-HMAC. + + For further details on the usage of these functions, see the sample + test application. diff --git a/SECURITY.md b/SECURITY.md index 1b871c415ec8fffe00c7a7b0f7fc68780a17a792..db6fcc950067c127b5f21686c8c3cb31fb8860b0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,12 +1,28 @@ # Security Policy +## Supported Versions + +Versions of the library that are currently being supported with security updates. + +| Version | Supported | Frameworks using this version | +| ------- | ------------------ | ----------------------------------- | +| 1.4 | :white_check_mark: | DPDK 23.07 | +| 1.3 | :white_check_mark: | DPDK 22.11, VPP | +| 1.2 | :x: | - | +| 1.1 | :x: | - | +| 1.0 | :white_check_mark: | DPDK 21.11 | +| < 1.0 | :x: | - | + ## Reporting a Vulnerability To report a vulnerability please send an email to the security team at ipsec-mb-security@arm.com. -If Intel-ipsec-mb is also affected, please follow steps from [Vulnerability Handling Guidelines](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). +If intel-ipsec-mb is also affected, please follow steps from [Vulnerability Handling Guidelines](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). + +If the vulnerability is accepted then an update will be developed and provided against reported library version. +Timeline for providing an update depends on development complexity. ## Security Considerations & Options for Increased Security -Please refer to [Section of README](https://gitlab.arm.com/arm-reference-solutions/ipsec-mb#6-security-considerations--options-for-increased-security) for the details on this topic. +Please refer to [Section of README](https://gitlab.arm.com/arm-reference-solutions/ipsec-mb#7-security-considerations--options-for-increased-security) for the details on this topic. diff --git a/cmake/clang-format.cmake b/cmake/clang-format.cmake new file mode 100644 index 0000000000000000000000000000000000000000..6d71f07cf866540bf4dee63dd318701aa76c191c --- /dev/null +++ b/cmake/clang-format.cmake @@ -0,0 +1,48 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# set clang-format binary name +if(NOT CLANG_FORMAT_BIN) + set(CLANG_FORMAT_BIN clang-format) +endif() + +find_program(CLANG_FORMAT NAMES ${CLANG_FORMAT_BIN}) + +# set up target if clang-format available +if(CLANG_FORMAT) + file(GLOB_RECURSE CLANG_FORMAT_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.[ch]") + + add_custom_target(style + COMMENT "Checking style using clang-format" + COMMAND clang-format -style=file --dry-run --Werror ${CLANG_FORMAT_SRC_FILES} + ) + + add_custom_target(style-fix + COMMENT "Fixing style issues using clang-format" + COMMAND clang-format -style=file -i ${CLANG_FORMAT_SRC_FILES} + ) +endif() + + diff --git a/cmake/utils.cmake b/cmake/utils.cmake new file mode 100644 index 0000000000000000000000000000000000000000..a42439013912332545a97504886579981e782b81 --- /dev/null +++ b/cmake/utils.cmake @@ -0,0 +1,277 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# extract library version from header file +macro(imb_get_version IMB_HDR_FILE) + file(STRINGS ${IMB_HDR_FILE} VER_STR REGEX "^.*IMB_VERSION_STR.*$") + string(REGEX MATCH "[0-9]+.[0-9]+.[0-9]+(-[a-z]+)?" IPSEC_MB_VERSION_FULL ${VER_STR}) + string(REGEX MATCH "[0-9]+.[0-9]+.[0-9]+" IPSEC_MB_VERSION ${IPSEC_MB_VERSION_FULL}) +endmacro() + +macro(imb_detect_os) + message(STATUS "OPERATING SYSTEM... ${CMAKE_HOST_SYSTEM_NAME}") + if(CMAKE_HOST_UNIX) + if(CMAKE_HOST_SYSTEM_NAME STREQUAL "FreeBSD") + set(FREEBSD 1) + else() + set(LINUX 1) + endif() + else() + set(WINDOWS 1) + endif() +endmacro() + +# set default project settings +macro(imb_set_proj_defaults) + # flag to force full project build + set(FULL_PROJECT_BUILD TRUE) + + # set default build type if not specified and not a multi-config generator + get_property(multi_config_gen GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if(NOT CMAKE_BUILD_TYPE + AND NOT CMAKE_CONFIGURATION_TYPES + AND NOT multi_config_gen) + set(CMAKE_BUILD_TYPE + "Release" + CACHE STRING "Selected build type") + endif() + + ####################################### + # set default library options + ####################################### + option(AESNI_EMU "AESNI emulation support" OFF) + option(SAFE_PARAM "API input parameter checking" ON) + option(SAFE_DATA "Sensitive data cleared from registers and memory at operation end" ON) + option(SAFE_LOOKUP "Lookups depending on sensitive data are constant time" ON) + option(SAFE_OPTIONS "Enable all safe options" ON) + option(BUILD_SHARED_LIBS "Build shared library" ON) + option(CMAKE_VERBOSE_MAKEFILE "Verbose build output" OFF) + set(EXTRA_CFLAGS "" CACHE STRING "Extra compiler flags") + + # disable all SAFE options when SAFE_OPTIONS false + if(NOT SAFE_OPTIONS) + message(STATUS "SAFE_OPTIONS disabled") + set(SAFE_PARAM OFF) + set(SAFE_DATA OFF) + set(SAFE_LOOKUP OFF) + endif() + + # project options list (used by print_help target) + set(IPSEC_MB_OPTIONS CMAKE_BUILD_TYPE IPSEC_MB_OPTIONS + AESNI_EMU SAFE_PARAM SAFE_DATA SAFE_LOOKUP + SAFE_OPTIONS BUILD_SHARED_LIBS + CMAKE_VERBOSE_MAKEFILE EXTRA_CFLAGS + ) + + # clear default release build C Compiler Flags + set(CMAKE_C_FLAGS_RELEASE "" CACHE STRING "" FORCE) + # clear default debug build C Compiler Flags + set(CMAKE_C_FLAGS_DEBUG "" CACHE STRING "" FORCE) + + ######################################## + # print build information + ######################################## + message(STATUS "AESNI emulation support... ${AESNI_EMU}") + message(STATUS "SAFE_OPTIONS... ${SAFE_OPTIONS}") + message(STATUS "SAFE_PARAM... ${SAFE_PARAM}") + message(STATUS "SAFE_DATA... ${SAFE_DATA}") + message(STATUS "SAFE_LOOKUP... ${SAFE_LOOKUP}") + message(STATUS "BUILD_SHARED_LIBS... ${BUILD_SHARED_LIBS}") + message(STATUS "CMAKE_GENERATOR... ${CMAKE_GENERATOR}") + if(${CMAKE_GENERATOR_PLATFORM}) + message(STATUS "GENERATOR PLATFORM... ${CMAKE_GENERATOR_PLATFORM}") + endif() + if (NOT multi_config_gen) + message(STATUS "BUILD_TYPE... ${CMAKE_BUILD_TYPE}") + message(STATUS "CMAKE_VERBOSE_MAKEFILE... ${CMAKE_VERBOSE_MAKEFILE}") + endif() + if (EXTRA_CFLAGS) + message(STATUS "EXTRA_CFLAGS... ${EXTRA_CFLAGS}") + endif() + +endmacro() + +# set binary output directory if specified +macro(imb_set_binary_dir) + # place all binaries in ${IMB_BIN_DIR} + if(IMB_BIN_DIR) + message(STATUS "BINARY DIRECTORY... ${IMB_BIN_DIR}") + get_property(multi_config_gen GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${IMB_BIN_DIR}) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${IMB_BIN_DIR}) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${IMB_BIN_DIR}) + endif() +endmacro() + +# compiler checks +macro(imb_compiler_check) + if((${CMAKE_C_COMPILER_ID} STREQUAL "GNU") AND + (CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0)) + message(FATAL_ERROR "GNU C Compiler version must be 5.0 or higher") + endif() +endmacro() + +# add uninstall target +macro(imb_add_target_uninstall UNINSTALL_ROUTINE) + configure_file(${UNINSTALL_ROUTINE} + "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" + IMMEDIATE @ONLY) + + add_custom_target(uninstall + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) +endmacro() + +# add print_help target +macro(imb_add_target_print_help OPTIONS) + add_custom_target( + print_help + COMMAND ${CMAKE_COMMAND} -E echo "Available build options:" + VERBATIM + ) + + foreach (OPTION ${OPTIONS}) + get_property(HELP_TEXT CACHE ${OPTION} PROPERTY HELPSTRING) + if(HELP_TEXT) + add_custom_command(TARGET print_help + COMMAND + ${CMAKE_COMMAND} -E echo " ${OPTION}=${${OPTION}} - ${HELP_TEXT}" + ) + endif() + endforeach() +endmacro() + +# style check using clang format +macro(imb_add_target_style_check) + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/clang-format.cmake") +endmacro() + +# add TAGS target +macro(imb_add_target_tags) + if(NOT WINDOWS) + add_custom_target( + TAGS + COMMAND ${CMAKE_COMMAND} -E echo "Building Tags table" + COMMAND bash -c "find . -name *.[ch] -print | etags -" + COMMAND bash -c "find ./ -name '*.asm' | etags -a -" + COMMAND bash -c "find ./ -name '*.inc' | etags -a -" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + VERBATIM + ) + endif() +endmacro() + +# add cppcheck targets +macro(imb_add_target_cppcheck_bughunt) + if(NOT WINDOWS) + # set cppcheck binary name + if(NOT CPPCHECK_BIN) + set(CPPCHECK_BIN cppcheck) + endif() + + find_program(CPPCHECK NAMES ${CPPCHECK_BIN}) + + # add targets if cppcheck available + if(CPPCHECK) + # output compilation database + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + execute_process( + COMMAND bash -c "getconf _NPROCESSORS_ONLN" + OUTPUT_VARIABLE nprocs + OUTPUT_STRIP_TRAILING_WHITESPACE) + + # set flags + set(CPPCHECK_FLAGS "-j ${nprocs}") + set(CPPCHECK_FLAGS1 "--cppcheck-build-dir=.cppcheck ${CPPCHECK_FLAGS}") + set(CPPCHECK_FLAGS2 "--cppcheck-build-dir=.bughunt ${CPPCHECK_FLAGS}") + + # add cppcheck target + add_custom_target( + cppcheck + COMMAND ${CMAKE_COMMAND} -E echo "Running cppcheck:" + COMMAND bash -c "mkdir -p .cppcheck" + COMMAND + bash -c + "${CPPCHECK} --force --enable=all ${CPPCHECK_FLAGS1} --project=./compile_commands.json" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM) + + # add bughunt target + add_custom_target( + bughunt + COMMAND ${CMAKE_COMMAND} -E echo "Running cppcheck bughunt:" + COMMAND bash -c "mkdir -p .bughunt" + COMMAND + bash -c + "${CPPCHECK} --bug-hunting --inconclusive ${CPPCHECK_FLAGS2} --project=./compile_commands.json" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM) + endif(CPPCHECK) + endif(NOT WINDOWS) +endmacro() + +# add spellcheck target +# +# Check spelling in the code with codespell. See +# https://github.com/codespell-project/codespell for more details. Codespell +# options explained: -d -- disable colours (emacs colours it anyway) -L +# -- List of words to be ignored -S -- skip file types -I FILE -- File +# containing words to be ignored +macro(imb_add_target_spellcheck) + # set cppcheck binary name + if(NOT CODESPELL_BIN) + set(CODESPELL_BIN codespell) + endif() + + find_program(CODESPELL NAMES ${CODESPELL_BIN}) + + # ignore some needed words + set(CS_IGNORE_WORDS "iinclude,struc,fo,ue,od,ba,padd") + + if(CODESPELL) + add_custom_target( + spellcheck + COMMAND ${CMAKE_COMMAND} -E echo "Running spellcheck:" + COMMAND + bash -c "${CODESPELL} -d -L ${CS_IGNORE_WORDS} \ + -S '*.obj,*.o,*.a,*.so,*.lib,*~,*.so,*.so.*,*.d,imb-perf' \ + -S 'imb-kat,imb-xvalid' \ + ./lib ./perf ./test README.md SECURITY.md CONTRIBUTING \ + Makefile win_x64.mak ReleaseNotes.txt LICENSE ${CS_EXTRA_OPTS}" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + VERBATIM) + endif() +endmacro() + +# add doxy target +macro(imb_add_target_doxy) + if(NOT WINDOWS) + add_custom_target( + doxy + COMMAND ${CMAKE_COMMAND} -E echo "Generating documentation..." + COMMAND bash -c "doxygen api_doxygen.conf" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/lib + VERBATIM) + endif() +endmacro() diff --git a/docs/README_8md.html b/docs/README_8md.html new file mode 100644 index 0000000000000000000000000000000000000000..31c738221d4f9aeb96ef370929bae8b61d516804 --- /dev/null +++ b/docs/README_8md.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: README.md File Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
README.md File Reference
+
+
+
+
+ + + + diff --git a/docs/annotated.html b/docs/annotated.html new file mode 100644 index 0000000000000000000000000000000000000000..bb0c3fbab50837b716e55fbcda9a5afc015b9a1d --- /dev/null +++ b/docs/annotated.html @@ -0,0 +1,124 @@ + + + + + + + +intel-ipsec-mb: Data Structures + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Data Structures
+
+
+
Here are the data structures with brief descriptions:
+ + + + + + + + + + +
 Cchacha20_poly1305_context_dataHolds Chacha20-Poly1305 operation context
 Cgcm_context_dataHolds GCM operation context
 Cgcm_key_dataHolds intermediate key data needed to improve performance
 CIMB_JOB
 CIMB_MGR
 CIMB_SGL_IOV
 Cimb_uint128_t
 Ckasumi_key_sched_s
 Csnow3g_key_schedule_s
+
+
+
+ + + + diff --git a/docs/bc_sd.png b/docs/bc_sd.png new file mode 100644 index 0000000000000000000000000000000000000000..31ca888dc71049713b35c351933a8d0f36180bf1 Binary files /dev/null and b/docs/bc_sd.png differ diff --git a/docs/classes.html b/docs/classes.html new file mode 100644 index 0000000000000000000000000000000000000000..bbe6406e5ed753b275e969e93d547c9a5cd2bc4f --- /dev/null +++ b/docs/classes.html @@ -0,0 +1,129 @@ + + + + + + + +intel-ipsec-mb: Data Structure Index + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Data Structure Index
+
+
+
C | G | I | K | S
+
+
+
C
+
chacha20_poly1305_context_data
+
+
G
+
gcm_context_data
gcm_key_data
+
+
I
+
IMB_JOB
IMB_MGR
IMB_SGL_IOV
imb_uint128_t
+
+
K
+
kasumi_key_sched_s
+
+
S
+
snow3g_key_schedule_s
+
+
+
+ + + + diff --git a/docs/doc.svg b/docs/doc.svg new file mode 100644 index 0000000000000000000000000000000000000000..0b928a531713b063e737da937bb85a53365a8ec2 --- /dev/null +++ b/docs/doc.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + diff --git a/docs/docd.svg b/docs/docd.svg new file mode 100644 index 0000000000000000000000000000000000000000..ac18b27552268ce7e82797636da773a6c5bbc4b8 --- /dev/null +++ b/docs/docd.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + diff --git a/docs/doxygen.css b/docs/doxygen.css new file mode 100644 index 0000000000000000000000000000000000000000..0caa19b274754811225e8b553e945df934742506 --- /dev/null +++ b/docs/doxygen.css @@ -0,0 +1,2017 @@ +/* The standard CSS for doxygen 1.9.7*/ + +html { +/* page base colors */ +--page-background-color: white; +--page-foreground-color: black; +--page-link-color: #3D578C; +--page-visited-link-color: #4665A2; + +/* index */ +--index-odd-item-bg-color: #F8F9FC; +--index-even-item-bg-color: white; +--index-header-color: black; +--index-separator-color: #A0A0A0; + +/* header */ +--header-background-color: #F9FAFC; +--header-separator-color: #C4CFE5; +--header-gradient-image: url('nav_h.png'); +--group-header-separator-color: #879ECB; +--group-header-color: #354C7B; +--inherit-header-color: gray; + +--footer-foreground-color: #2A3D61; +--footer-logo-width: 104px; +--citation-label-color: #334975; +--glow-color: cyan; + +--title-background-color: white; +--title-separator-color: #5373B4; +--directory-separator-color: #9CAFD4; +--separator-color: #4A6AAA; + +--blockquote-background-color: #F7F8FB; +--blockquote-border-color: #9CAFD4; + +--scrollbar-thumb-color: #9CAFD4; +--scrollbar-background-color: #F9FAFC; + +--icon-background-color: #728DC1; +--icon-foreground-color: white; +--icon-doc-image: url('doc.svg'); +--icon-folder-open-image: url('folderopen.svg'); +--icon-folder-closed-image: url('folderclosed.svg'); + +/* brief member declaration list */ +--memdecl-background-color: #F9FAFC; +--memdecl-separator-color: #DEE4F0; +--memdecl-foreground-color: #555; +--memdecl-template-color: #4665A2; + +/* detailed member list */ +--memdef-border-color: #A8B8D9; +--memdef-title-background-color: #E2E8F2; +--memdef-title-gradient-image: url('nav_f.png'); +--memdef-proto-background-color: #DFE5F1; +--memdef-proto-text-color: #253555; +--memdef-proto-text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); +--memdef-doc-background-color: white; +--memdef-param-name-color: #602020; +--memdef-template-color: #4665A2; + +/* tables */ +--table-cell-border-color: #2D4068; +--table-header-background-color: #374F7F; +--table-header-foreground-color: #FFFFFF; + +/* labels */ +--label-background-color: #728DC1; +--label-left-top-border-color: #5373B4; +--label-right-bottom-border-color: #C4CFE5; +--label-foreground-color: white; + +/** navigation bar/tree/menu */ +--nav-background-color: #F9FAFC; +--nav-foreground-color: #364D7C; +--nav-gradient-image: url('tab_b.png'); +--nav-gradient-hover-image: url('tab_h.png'); +--nav-gradient-active-image: url('tab_a.png'); +--nav-gradient-active-image-parent: url("../tab_a.png"); +--nav-separator-image: url('tab_s.png'); +--nav-breadcrumb-image: url('bc_s.png'); +--nav-breadcrumb-border-color: #C2CDE4; +--nav-splitbar-image: url('splitbar.png'); +--nav-font-size-level1: 13px; +--nav-font-size-level2: 10px; +--nav-font-size-level3: 9px; +--nav-text-normal-color: #283A5D; +--nav-text-hover-color: white; +--nav-text-active-color: white; +--nav-text-normal-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); +--nav-text-hover-shadow: 0px 1px 1px rgba(0, 0, 0, 1.0); +--nav-text-active-shadow: 0px 1px 1px rgba(0, 0, 0, 1.0); +--nav-menu-button-color: #364D7C; +--nav-menu-background-color: white; +--nav-menu-foreground-color: #555555; +--nav-menu-toggle-color: rgba(255, 255, 255, 0.5); +--nav-arrow-color: #9CAFD4; +--nav-arrow-selected-color: #9CAFD4; + +/* table of contents */ +--toc-background-color: #F4F6FA; +--toc-border-color: #D8DFEE; +--toc-header-color: #4665A2; +--toc-down-arrow-image: url("data:image/svg+xml;utf8,&%238595;"); + +/** search field */ +--search-background-color: white; +--search-foreground-color: #909090; +--search-magnification-image: url('mag.svg'); +--search-magnification-select-image: url('mag_sel.svg'); +--search-active-color: black; +--search-filter-background-color: #F9FAFC; +--search-filter-foreground-color: black; +--search-filter-border-color: #90A5CE; +--search-filter-highlight-text-color: white; +--search-filter-highlight-bg-color: #3D578C; +--search-results-foreground-color: #425E97; +--search-results-background-color: #EEF1F7; +--search-results-border-color: black; +--search-box-shadow: inset 0.5px 0.5px 3px 0px #555; + +/** code fragments */ +--code-keyword-color: #008000; +--code-type-keyword-color: #604020; +--code-flow-keyword-color: #E08000; +--code-comment-color: #800000; +--code-preprocessor-color: #806020; +--code-string-literal-color: #002080; +--code-char-literal-color: #008080; +--code-xml-cdata-color: black; +--code-vhdl-digit-color: #FF00FF; +--code-vhdl-char-color: #000000; +--code-vhdl-keyword-color: #700070; +--code-vhdl-logic-color: #FF0000; +--code-link-color: #4665A2; +--code-external-link-color: #4665A2; +--fragment-foreground-color: black; +--fragment-background-color: #FBFCFD; +--fragment-border-color: #C4CFE5; +--fragment-lineno-border-color: #00FF00; +--fragment-lineno-background-color: #E8E8E8; +--fragment-lineno-foreground-color: black; +--fragment-lineno-link-fg-color: #4665A2; +--fragment-lineno-link-bg-color: #D8D8D8; +--fragment-lineno-link-hover-fg-color: #4665A2; +--fragment-lineno-link-hover-bg-color: #C8C8C8; +--tooltip-foreground-color: black; +--tooltip-background-color: white; +--tooltip-border-color: gray; +--tooltip-doc-color: grey; +--tooltip-declaration-color: #006318; +--tooltip-link-color: #4665A2; +--tooltip-shadow: 1px 1px 7px gray; + +/** font-family */ +--font-family-normal: Roboto,sans-serif; +--font-family-monospace: 'JetBrains Mono',Consolas,Monaco,'Andale Mono','Ubuntu Mono',monospace,fixed; +--font-family-nav: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif; +--font-family-title: Tahoma,Arial,sans-serif; +--font-family-toc: Verdana,'DejaVu Sans',Geneva,sans-serif; +--font-family-search: Arial,Verdana,sans-serif; +--font-family-icon: Arial,Helvetica; +--font-family-tooltip: Roboto,sans-serif; + +} + +@media (prefers-color-scheme: dark) { + html:not(.dark-mode) { + color-scheme: dark; + +/* page base colors */ +--page-background-color: black; +--page-foreground-color: #C9D1D9; +--page-link-color: #90A5CE; +--page-visited-link-color: #A3B4D7; + +/* index */ +--index-odd-item-bg-color: #0B101A; +--index-even-item-bg-color: black; +--index-header-color: #C4CFE5; +--index-separator-color: #334975; + +/* header */ +--header-background-color: #070B11; +--header-separator-color: #141C2E; +--header-gradient-image: url('nav_hd.png'); +--group-header-separator-color: #283A5D; +--group-header-color: #90A5CE; +--inherit-header-color: #A0A0A0; + +--footer-foreground-color: #5B7AB7; +--footer-logo-width: 60px; +--citation-label-color: #90A5CE; +--glow-color: cyan; + +--title-background-color: #090D16; +--title-separator-color: #354C79; +--directory-separator-color: #283A5D; +--separator-color: #283A5D; + +--blockquote-background-color: #101826; +--blockquote-border-color: #283A5D; + +--scrollbar-thumb-color: #283A5D; +--scrollbar-background-color: #070B11; + +--icon-background-color: #334975; +--icon-foreground-color: #C4CFE5; +--icon-doc-image: url('docd.svg'); +--icon-folder-open-image: url('folderopend.svg'); +--icon-folder-closed-image: url('folderclosedd.svg'); + +/* brief member declaration list */ +--memdecl-background-color: #0B101A; +--memdecl-separator-color: #2C3F65; +--memdecl-foreground-color: #BBB; +--memdecl-template-color: #7C95C6; + +/* detailed member list */ +--memdef-border-color: #233250; +--memdef-title-background-color: #1B2840; +--memdef-title-gradient-image: url('nav_fd.png'); +--memdef-proto-background-color: #19243A; +--memdef-proto-text-color: #9DB0D4; +--memdef-proto-text-shadow: 0px 1px 1px rgba(0, 0, 0, 0.9); +--memdef-doc-background-color: black; +--memdef-param-name-color: #D28757; +--memdef-template-color: #7C95C6; + +/* tables */ +--table-cell-border-color: #283A5D; +--table-header-background-color: #283A5D; +--table-header-foreground-color: #C4CFE5; + +/* labels */ +--label-background-color: #354C7B; +--label-left-top-border-color: #4665A2; +--label-right-bottom-border-color: #283A5D; +--label-foreground-color: #CCCCCC; + +/** navigation bar/tree/menu */ +--nav-background-color: #101826; +--nav-foreground-color: #364D7C; +--nav-gradient-image: url('tab_bd.png'); +--nav-gradient-hover-image: url('tab_hd.png'); +--nav-gradient-active-image: url('tab_ad.png'); +--nav-gradient-active-image-parent: url("../tab_ad.png"); +--nav-separator-image: url('tab_sd.png'); +--nav-breadcrumb-image: url('bc_sd.png'); +--nav-breadcrumb-border-color: #2A3D61; +--nav-splitbar-image: url('splitbard.png'); +--nav-font-size-level1: 13px; +--nav-font-size-level2: 10px; +--nav-font-size-level3: 9px; +--nav-text-normal-color: #B6C4DF; +--nav-text-hover-color: #DCE2EF; +--nav-text-active-color: #DCE2EF; +--nav-text-normal-shadow: 0px 1px 1px black; +--nav-text-hover-shadow: 0px 1px 1px rgba(0, 0, 0, 1.0); +--nav-text-active-shadow: 0px 1px 1px rgba(0, 0, 0, 1.0); +--nav-menu-button-color: #B6C4DF; +--nav-menu-background-color: #05070C; +--nav-menu-foreground-color: #BBBBBB; +--nav-menu-toggle-color: rgba(255, 255, 255, 0.2); +--nav-arrow-color: #334975; +--nav-arrow-selected-color: #90A5CE; + +/* table of contents */ +--toc-background-color: #151E30; +--toc-border-color: #202E4A; +--toc-header-color: #A3B4D7; +--toc-down-arrow-image: url("data:image/svg+xml;utf8,&%238595;"); + +/** search field */ +--search-background-color: black; +--search-foreground-color: #C5C5C5; +--search-magnification-image: url('mag_d.svg'); +--search-magnification-select-image: url('mag_seld.svg'); +--search-active-color: #C5C5C5; +--search-filter-background-color: #101826; +--search-filter-foreground-color: #90A5CE; +--search-filter-border-color: #7C95C6; +--search-filter-highlight-text-color: #BCC9E2; +--search-filter-highlight-bg-color: #283A5D; +--search-results-background-color: #101826; +--search-results-foreground-color: #90A5CE; +--search-results-border-color: #7C95C6; +--search-box-shadow: inset 0.5px 0.5px 3px 0px #2F436C; + +/** code fragments */ +--code-keyword-color: #CC99CD; +--code-type-keyword-color: #AB99CD; +--code-flow-keyword-color: #E08000; +--code-comment-color: #717790; +--code-preprocessor-color: #65CABE; +--code-string-literal-color: #7EC699; +--code-char-literal-color: #00E0F0; +--code-xml-cdata-color: #C9D1D9; +--code-vhdl-digit-color: #FF00FF; +--code-vhdl-char-color: #000000; +--code-vhdl-keyword-color: #700070; +--code-vhdl-logic-color: #FF0000; +--code-link-color: #79C0FF; +--code-external-link-color: #79C0FF; +--fragment-foreground-color: #C9D1D9; +--fragment-background-color: black; +--fragment-border-color: #30363D; +--fragment-lineno-border-color: #30363D; +--fragment-lineno-background-color: black; +--fragment-lineno-foreground-color: #6E7681; +--fragment-lineno-link-fg-color: #6E7681; +--fragment-lineno-link-bg-color: #303030; +--fragment-lineno-link-hover-fg-color: #8E96A1; +--fragment-lineno-link-hover-bg-color: #505050; +--tooltip-foreground-color: #C9D1D9; +--tooltip-background-color: #202020; +--tooltip-border-color: #C9D1D9; +--tooltip-doc-color: #D9E1E9; +--tooltip-declaration-color: #20C348; +--tooltip-link-color: #79C0FF; +--tooltip-shadow: none; + +/** font-family */ +--font-family-normal: Roboto,sans-serif; +--font-family-monospace: 'JetBrains Mono',Consolas,Monaco,'Andale Mono','Ubuntu Mono',monospace,fixed; +--font-family-nav: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif; +--font-family-title: Tahoma,Arial,sans-serif; +--font-family-toc: Verdana,'DejaVu Sans',Geneva,sans-serif; +--font-family-search: Arial,Verdana,sans-serif; +--font-family-icon: Arial,Helvetica; +--font-family-tooltip: Roboto,sans-serif; + +}} +body { + background-color: var(--page-background-color); + color: var(--page-foreground-color); +} + +body, table, div, p, dl { + font-weight: 400; + font-size: 14px; + font-family: var(--font-family-normal); + line-height: 22px; +} + +/* @group Heading Levels */ + +.title { + font-weight: 400; + font-size: 14px; + font-family: var(--font-family-normal); + line-height: 28px; + font-size: 150%; + font-weight: bold; + margin: 10px 2px; +} + +h1.groupheader { + font-size: 150%; +} + +h2.groupheader { + border-bottom: 1px solid var(--group-header-separator-color); + color: var(--group-header-color); + font-size: 150%; + font-weight: normal; + margin-top: 1.75em; + padding-top: 8px; + padding-bottom: 4px; + width: 100%; +} + +h3.groupheader { + font-size: 100%; +} + +h1, h2, h3, h4, h5, h6 { + -webkit-transition: text-shadow 0.5s linear; + -moz-transition: text-shadow 0.5s linear; + -ms-transition: text-shadow 0.5s linear; + -o-transition: text-shadow 0.5s linear; + transition: text-shadow 0.5s linear; + margin-right: 15px; +} + +h1.glow, h2.glow, h3.glow, h4.glow, h5.glow, h6.glow { + text-shadow: 0 0 15px var(--glow-color); +} + +dt { + font-weight: bold; +} + +p.startli, p.startdd { + margin-top: 2px; +} + +th p.starttd, th p.intertd, th p.endtd { + font-size: 100%; + font-weight: 700; +} + +p.starttd { + margin-top: 0px; +} + +p.endli { + margin-bottom: 0px; +} + +p.enddd { + margin-bottom: 4px; +} + +p.endtd { + margin-bottom: 2px; +} + +p.interli { +} + +p.interdd { +} + +p.intertd { +} + +/* @end */ + +caption { + font-weight: bold; +} + +span.legend { + font-size: 70%; + text-align: center; +} + +h3.version { + font-size: 90%; + text-align: center; +} + +div.navtab { + padding-right: 15px; + text-align: right; + line-height: 110%; +} + +div.navtab table { + border-spacing: 0; +} + +td.navtab { + padding-right: 6px; + padding-left: 6px; +} + +td.navtabHL { + background-image: var(--nav-gradient-active-image); + background-repeat:repeat-x; + padding-right: 6px; + padding-left: 6px; +} + +td.navtabHL a, td.navtabHL a:visited { + color: var(--nav-text-hover-color); + text-shadow: var(--nav-text-hover-shadow); +} + +a.navtab { + font-weight: bold; +} + +div.qindex{ + text-align: center; + width: 100%; + line-height: 140%; + font-size: 130%; + color: var(--index-separator-color); +} + +dt.alphachar{ + font-size: 180%; + font-weight: bold; +} + +.alphachar a{ + color: var(--index-header-color); +} + +.alphachar a:hover, .alphachar a:visited{ + text-decoration: none; +} + +.classindex dl { + padding: 25px; + column-count:1 +} + +.classindex dd { + display:inline-block; + margin-left: 50px; + width: 90%; + line-height: 1.15em; +} + +.classindex dl.even { + background-color: var(--index-even-item-bg-color); +} + +.classindex dl.odd { + background-color: var(--index-odd-item-bg-color); +} + +@media(min-width: 1120px) { + .classindex dl { + column-count:2 + } +} + +@media(min-width: 1320px) { + .classindex dl { + column-count:3 + } +} + + +/* @group Link Styling */ + +a { + color: var(--page-link-color); + font-weight: normal; + text-decoration: none; +} + +.contents a:visited { + color: var(--page-visited-link-color); +} + +a:hover { + text-decoration: underline; +} + +a.el { + font-weight: bold; +} + +a.elRef { +} + +a.code, a.code:visited, a.line, a.line:visited { + color: var(--code-link-color); +} + +a.codeRef, a.codeRef:visited, a.lineRef, a.lineRef:visited { + color: var(--code-external-link-color); +} + +a.code.hl_class { /* style for links to class names in code snippets */ } +a.code.hl_struct { /* style for links to struct names in code snippets */ } +a.code.hl_union { /* style for links to union names in code snippets */ } +a.code.hl_interface { /* style for links to interface names in code snippets */ } +a.code.hl_protocol { /* style for links to protocol names in code snippets */ } +a.code.hl_category { /* style for links to category names in code snippets */ } +a.code.hl_exception { /* style for links to exception names in code snippets */ } +a.code.hl_service { /* style for links to service names in code snippets */ } +a.code.hl_singleton { /* style for links to singleton names in code snippets */ } +a.code.hl_concept { /* style for links to concept names in code snippets */ } +a.code.hl_namespace { /* style for links to namespace names in code snippets */ } +a.code.hl_package { /* style for links to package names in code snippets */ } +a.code.hl_define { /* style for links to macro names in code snippets */ } +a.code.hl_function { /* style for links to function names in code snippets */ } +a.code.hl_variable { /* style for links to variable names in code snippets */ } +a.code.hl_typedef { /* style for links to typedef names in code snippets */ } +a.code.hl_enumvalue { /* style for links to enum value names in code snippets */ } +a.code.hl_enumeration { /* style for links to enumeration names in code snippets */ } +a.code.hl_signal { /* style for links to Qt signal names in code snippets */ } +a.code.hl_slot { /* style for links to Qt slot names in code snippets */ } +a.code.hl_friend { /* style for links to friend names in code snippets */ } +a.code.hl_dcop { /* style for links to KDE3 DCOP names in code snippets */ } +a.code.hl_property { /* style for links to property names in code snippets */ } +a.code.hl_event { /* style for links to event names in code snippets */ } +a.code.hl_sequence { /* style for links to sequence names in code snippets */ } +a.code.hl_dictionary { /* style for links to dictionary names in code snippets */ } + +/* @end */ + +dl.el { + margin-left: -1cm; +} + +ul { + overflow: visible; +} + +ul.multicol { + -moz-column-gap: 1em; + -webkit-column-gap: 1em; + column-gap: 1em; + -moz-column-count: 3; + -webkit-column-count: 3; + column-count: 3; + list-style-type: none; +} + +#side-nav ul { + overflow: visible; /* reset ul rule for scroll bar in GENERATE_TREEVIEW window */ +} + +#main-nav ul { + overflow: visible; /* reset ul rule for the navigation bar drop down lists */ +} + +.fragment { + text-align: left; + direction: ltr; + overflow-x: auto; /*Fixed: fragment lines overlap floating elements*/ + overflow-y: hidden; +} + +pre.fragment { + border: 1px solid var(--fragment-border-color); + background-color: var(--fragment-background-color); + color: var(--fragment-foreground-color); + padding: 4px 6px; + margin: 4px 8px 4px 2px; + overflow: auto; + word-wrap: break-word; + font-size: 9pt; + line-height: 125%; + font-family: var(--font-family-monospace); + font-size: 105%; +} + +div.fragment { + padding: 0 0 1px 0; /*Fixed: last line underline overlap border*/ + margin: 4px 8px 4px 2px; + color: var(--fragment-foreground-color); + background-color: var(--fragment-background-color); + border: 1px solid var(--fragment-border-color); +} + +div.line { + font-family: var(--font-family-monospace); + font-size: 13px; + min-height: 13px; + line-height: 1.2; + text-wrap: unrestricted; + white-space: -moz-pre-wrap; /* Moz */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: pre-wrap; /* CSS3 */ + word-wrap: break-word; /* IE 5.5+ */ + text-indent: -53px; + padding-left: 53px; + padding-bottom: 0px; + margin: 0px; + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +div.line:after { + content:"\000A"; + white-space: pre; +} + +div.line.glow { + background-color: var(--glow-color); + box-shadow: 0 0 10px var(--glow-color); +} + + +span.lineno { + padding-right: 4px; + margin-right: 9px; + text-align: right; + border-right: 2px solid var(--fragment-lineno-border-color); + color: var(--fragment-lineno-foreground-color); + background-color: var(--fragment-lineno-background-color); + white-space: pre; +} +span.lineno a, span.lineno a:visited { + color: var(--fragment-lineno-link-fg-color); + background-color: var(--fragment-lineno-link-bg-color); +} + +span.lineno a:hover { + color: var(--fragment-lineno-link-hover-fg-color); + background-color: var(--fragment-lineno-link-hover-bg-color); +} + +.lineno { + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +div.classindex ul { + list-style: none; + padding-left: 0; +} + +div.classindex span.ai { + display: inline-block; +} + +div.groupHeader { + margin-left: 16px; + margin-top: 12px; + font-weight: bold; +} + +div.groupText { + margin-left: 16px; + font-style: italic; +} + +body { + color: var(--page-foreground-color); + margin: 0; +} + +div.contents { + margin-top: 10px; + margin-left: 12px; + margin-right: 8px; +} + +p.formulaDsp { + text-align: center; +} + +img.dark-mode-visible { + display: none; +} +img.light-mode-visible { + display: none; +} + +img.formulaDsp { + +} + +img.formulaInl, img.inline { + vertical-align: middle; +} + +div.center { + text-align: center; + margin-top: 0px; + margin-bottom: 0px; + padding: 0px; +} + +div.center img { + border: 0px; +} + +address.footer { + text-align: right; + padding-right: 12px; +} + +img.footer { + border: 0px; + vertical-align: middle; + width: var(--footer-logo-width); +} + +.compoundTemplParams { + color: var(--memdecl-template-color); + font-size: 80%; + line-height: 120%; +} + +/* @group Code Colorization */ + +span.keyword { + color: var(--code-keyword-color); +} + +span.keywordtype { + color: var(--code-type-keyword-color); +} + +span.keywordflow { + color: var(--code-flow-keyword-color); +} + +span.comment { + color: var(--code-comment-color); +} + +span.preprocessor { + color: var(--code-preprocessor-color); +} + +span.stringliteral { + color: var(--code-string-literal-color); +} + +span.charliteral { + color: var(--code-char-literal-color); +} + +span.xmlcdata { + color: var(--code-xml-cdata-color); +} + +span.vhdldigit { + color: var(--code-vhdl-digit-color); +} + +span.vhdlchar { + color: var(--code-vhdl-char-color); +} + +span.vhdlkeyword { + color: var(--code-vhdl-keyword-color); +} + +span.vhdllogic { + color: var(--code-vhdl-logic-color); +} + +blockquote { + background-color: var(--blockquote-background-color); + border-left: 2px solid var(--blockquote-border-color); + margin: 0 24px 0 4px; + padding: 0 12px 0 16px; +} + +/* @end */ + +td.tiny { + font-size: 75%; +} + +.dirtab { + padding: 4px; + border-collapse: collapse; + border: 1px solid var(--table-cell-border-color); +} + +th.dirtab { + background-color: var(--table-header-background-color); + color: var(--table-header-foreground-color); + font-weight: bold; +} + +hr { + height: 0px; + border: none; + border-top: 1px solid var(--separator-color); +} + +hr.footer { + height: 1px; +} + +/* @group Member Descriptions */ + +table.memberdecls { + border-spacing: 0px; + padding: 0px; +} + +.memberdecls td, .fieldtable tr { + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +.memberdecls td.glow, .fieldtable tr.glow { + background-color: var(--glow-color); + box-shadow: 0 0 15px var(--glow-color); +} + +.mdescLeft, .mdescRight, +.memItemLeft, .memItemRight, +.memTemplItemLeft, .memTemplItemRight, .memTemplParams { + background-color: var(--memdecl-background-color); + border: none; + margin: 4px; + padding: 1px 0 0 8px; +} + +.mdescLeft, .mdescRight { + padding: 0px 8px 4px 8px; + color: var(--memdecl-foreground-color); +} + +.memSeparator { + border-bottom: 1px solid var(--memdecl-separator-color); + line-height: 1px; + margin: 0px; + padding: 0px; +} + +.memItemLeft, .memTemplItemLeft { + white-space: nowrap; +} + +.memItemRight, .memTemplItemRight { + width: 100%; +} + +.memTemplParams { + color: var(--memdecl-template-color); + white-space: nowrap; + font-size: 80%; +} + +/* @end */ + +/* @group Member Details */ + +/* Styles for detailed member documentation */ + +.memtitle { + padding: 8px; + border-top: 1px solid var(--memdef-border-color); + border-left: 1px solid var(--memdef-border-color); + border-right: 1px solid var(--memdef-border-color); + border-top-right-radius: 4px; + border-top-left-radius: 4px; + margin-bottom: -1px; + background-image: var(--memdef-title-gradient-image); + background-repeat: repeat-x; + background-color: var(--memdef-title-background-color); + line-height: 1.25; + font-weight: 300; + float:left; +} + +.permalink +{ + font-size: 65%; + display: inline-block; + vertical-align: middle; +} + +.memtemplate { + font-size: 80%; + color: var(--memdef-template-color); + font-weight: normal; + margin-left: 9px; +} + +.mempage { + width: 100%; +} + +.memitem { + padding: 0; + margin-bottom: 10px; + margin-right: 5px; + -webkit-transition: box-shadow 0.5s linear; + -moz-transition: box-shadow 0.5s linear; + -ms-transition: box-shadow 0.5s linear; + -o-transition: box-shadow 0.5s linear; + transition: box-shadow 0.5s linear; + display: table !important; + width: 100%; +} + +.memitem.glow { + box-shadow: 0 0 15px var(--glow-color); +} + +.memname { + font-weight: 400; + margin-left: 6px; +} + +.memname td { + vertical-align: bottom; +} + +.memproto, dl.reflist dt { + border-top: 1px solid var(--memdef-border-color); + border-left: 1px solid var(--memdef-border-color); + border-right: 1px solid var(--memdef-border-color); + padding: 6px 0px 6px 0px; + color: var(--memdef-proto-text-color); + font-weight: bold; + text-shadow: var(--memdef-proto-text-shadow); + background-color: var(--memdef-proto-background-color); + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + border-top-right-radius: 4px; +} + +.overload { + font-family: var(--font-family-monospace); + font-size: 65%; +} + +.memdoc, dl.reflist dd { + border-bottom: 1px solid var(--memdef-border-color); + border-left: 1px solid var(--memdef-border-color); + border-right: 1px solid var(--memdef-border-color); + padding: 6px 10px 2px 10px; + border-top-width: 0; + background-image:url('nav_g.png'); + background-repeat:repeat-x; + background-color: var(--memdef-doc-background-color); + /* opera specific markup */ + border-bottom-left-radius: 4px; + border-bottom-right-radius: 4px; + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + /* firefox specific markup */ + -moz-border-radius-bottomleft: 4px; + -moz-border-radius-bottomright: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + /* webkit specific markup */ + -webkit-border-bottom-left-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); +} + +dl.reflist dt { + padding: 5px; +} + +dl.reflist dd { + margin: 0px 0px 10px 0px; + padding: 5px; +} + +.paramkey { + text-align: right; +} + +.paramtype { + white-space: nowrap; +} + +.paramname { + color: var(--memdef-param-name-color); + white-space: nowrap; +} +.paramname em { + font-style: normal; +} +.paramname code { + line-height: 14px; +} + +.params, .retval, .exception, .tparams { + margin-left: 0px; + padding-left: 0px; +} + +.params .paramname, .retval .paramname, .tparams .paramname, .exception .paramname { + font-weight: bold; + vertical-align: top; +} + +.params .paramtype, .tparams .paramtype { + font-style: italic; + vertical-align: top; +} + +.params .paramdir, .tparams .paramdir { + font-family: var(--font-family-monospace); + vertical-align: top; +} + +table.mlabels { + border-spacing: 0px; +} + +td.mlabels-left { + width: 100%; + padding: 0px; +} + +td.mlabels-right { + vertical-align: bottom; + padding: 0px; + white-space: nowrap; +} + +span.mlabels { + margin-left: 8px; +} + +span.mlabel { + background-color: var(--label-background-color); + border-top:1px solid var(--label-left-top-border-color); + border-left:1px solid var(--label-left-top-border-color); + border-right:1px solid var(--label-right-bottom-border-color); + border-bottom:1px solid var(--label-right-bottom-border-color); + text-shadow: none; + color: var(--label-foreground-color); + margin-right: 4px; + padding: 2px 3px; + border-radius: 3px; + font-size: 7pt; + white-space: nowrap; + vertical-align: middle; +} + + + +/* @end */ + +/* these are for tree view inside a (index) page */ + +div.directory { + margin: 10px 0px; + border-top: 1px solid var(--directory-separator-color); + border-bottom: 1px solid var(--directory-separator-color); + width: 100%; +} + +.directory table { + border-collapse:collapse; +} + +.directory td { + margin: 0px; + padding: 0px; + vertical-align: top; +} + +.directory td.entry { + white-space: nowrap; + padding-right: 6px; + padding-top: 3px; +} + +.directory td.entry a { + outline:none; +} + +.directory td.entry a img { + border: none; +} + +.directory td.desc { + width: 100%; + padding-left: 6px; + padding-right: 6px; + padding-top: 3px; + border-left: 1px solid rgba(0,0,0,0.05); +} + +.directory tr.odd { + padding-left: 6px; + background-color: var(--index-odd-item-bg-color); +} + +.directory tr.even { + padding-left: 6px; + background-color: var(--index-even-item-bg-color); +} + +.directory img { + vertical-align: -30%; +} + +.directory .levels { + white-space: nowrap; + width: 100%; + text-align: right; + font-size: 9pt; +} + +.directory .levels span { + cursor: pointer; + padding-left: 2px; + padding-right: 2px; + color: var(--page-link-color); +} + +.arrow { + color: var(--nav-arrow-color); + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + cursor: pointer; + font-size: 80%; + display: inline-block; + width: 16px; + height: 22px; +} + +.icon { + font-family: var(--font-family-icon); + line-height: normal; + font-weight: bold; + font-size: 12px; + height: 14px; + width: 16px; + display: inline-block; + background-color: var(--icon-background-color); + color: var(--icon-foreground-color); + text-align: center; + border-radius: 4px; + margin-left: 2px; + margin-right: 2px; +} + +.icona { + width: 24px; + height: 22px; + display: inline-block; +} + +.iconfopen { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:var(--icon-folder-open-image); + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.iconfclosed { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:var(--icon-folder-closed-image); + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.icondoc { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:var(--icon-doc-image); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +/* @end */ + +div.dynheader { + margin-top: 8px; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +address { + font-style: normal; + color: var(--footer-foreground-color); +} + +table.doxtable caption { + caption-side: top; +} + +table.doxtable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.doxtable td, table.doxtable th { + border: 1px solid var(--table-cell-border-color); + padding: 3px 7px 2px; +} + +table.doxtable th { + background-color: var(--table-header-background-color); + color: var(--table-header-foreground-color); + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +table.fieldtable { + margin-bottom: 10px; + border: 1px solid var(--memdef-border-color); + border-spacing: 0px; + border-radius: 4px; + box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); +} + +.fieldtable td, .fieldtable th { + padding: 3px 7px 2px; +} + +.fieldtable td.fieldtype, .fieldtable td.fieldname { + white-space: nowrap; + border-right: 1px solid var(--memdef-border-color); + border-bottom: 1px solid var(--memdef-border-color); + vertical-align: top; +} + +.fieldtable td.fieldname { + padding-top: 3px; +} + +.fieldtable td.fielddoc { + border-bottom: 1px solid var(--memdef-border-color); +} + +.fieldtable td.fielddoc p:first-child { + margin-top: 0px; +} + +.fieldtable td.fielddoc p:last-child { + margin-bottom: 2px; +} + +.fieldtable tr:last-child td { + border-bottom: none; +} + +.fieldtable th { + background-image: var(--memdef-title-gradient-image); + background-repeat:repeat-x; + background-color: var(--memdef-title-background-color); + font-size: 90%; + color: var(--memdef-proto-text-color); + padding-bottom: 4px; + padding-top: 5px; + text-align:left; + font-weight: 400; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 1px solid var(--memdef-border-color); +} + + +.tabsearch { + top: 0px; + left: 10px; + height: 36px; + background-image: var(--nav-gradient-image); + z-index: 101; + overflow: hidden; + font-size: 13px; +} + +.navpath ul +{ + font-size: 11px; + background-image: var(--nav-gradient-image); + background-repeat:repeat-x; + background-position: 0 -5px; + height:30px; + line-height:30px; + color:var(--nav-text-normal-color); + border:solid 1px var(--nav-breadcrumb-border-color); + overflow:hidden; + margin:0px; + padding:0px; +} + +.navpath li +{ + list-style-type:none; + float:left; + padding-left:10px; + padding-right:15px; + background-image:var(--nav-breadcrumb-image); + background-repeat:no-repeat; + background-position:right; + color: var(--nav-foreground-color); +} + +.navpath li.navelem a +{ + height:32px; + display:block; + text-decoration: none; + outline: none; + color: var(--nav-text-normal-color); + font-family: var(--font-family-nav); + text-shadow: var(--nav-text-normal-shadow); + text-decoration: none; +} + +.navpath li.navelem a:hover +{ + color: var(--nav-text-hover-color); + text-shadow: var(--nav-text-hover-shadow); +} + +.navpath li.footer +{ + list-style-type:none; + float:right; + padding-left:10px; + padding-right:15px; + background-image:none; + background-repeat:no-repeat; + background-position:right; + color: var(--footer-foreground-color); + font-size: 8pt; +} + + +div.summary +{ + float: right; + font-size: 8pt; + padding-right: 5px; + width: 50%; + text-align: right; +} + +div.summary a +{ + white-space: nowrap; +} + +table.classindex +{ + margin: 10px; + white-space: nowrap; + margin-left: 3%; + margin-right: 3%; + width: 94%; + border: 0; + border-spacing: 0; + padding: 0; +} + +div.ingroups +{ + font-size: 8pt; + width: 50%; + text-align: left; +} + +div.ingroups a +{ + white-space: nowrap; +} + +div.header +{ + background-image: var(--header-gradient-image); + background-repeat:repeat-x; + background-color: var(--header-background-color); + margin: 0px; + border-bottom: 1px solid var(--header-separator-color); +} + +div.headertitle +{ + padding: 5px 5px 5px 10px; +} + +.PageDocRTL-title div.headertitle { + text-align: right; + direction: rtl; +} + +dl { + padding: 0 0 0 0; +} + +/* dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, dl.deprecated, dl.todo, dl.test, dl.bug, dl.examples */ +dl.section { + margin-left: 0px; + padding-left: 0px; +} + +dl.note { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #D0C000; +} + +dl.warning, dl.attention { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #FF0000; +} + +dl.pre, dl.post, dl.invariant { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #00D000; +} + +dl.deprecated { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #505050; +} + +dl.todo { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #00C0E0; +} + +dl.test { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #3030E0; +} + +dl.bug { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #C08050; +} + +dl.section dd { + margin-bottom: 6px; +} + + +#projectrow +{ + height: 56px; +} + +#projectlogo +{ + text-align: center; + vertical-align: bottom; + border-collapse: separate; +} + +#projectlogo img +{ + border: 0px none; +} + +#projectalign +{ + vertical-align: middle; + padding-left: 0.5em; +} + +#projectname +{ + font-size: 200%; + font-family: var(--font-family-title); + margin: 0px; + padding: 2px 0px; +} + +#projectbrief +{ + font-size: 90%; + font-family: var(--font-family-title); + margin: 0px; + padding: 0px; +} + +#projectnumber +{ + font-size: 50%; + font-family: 50% var(--font-family-title); + margin: 0px; + padding: 0px; +} + +#titlearea +{ + padding: 0px; + margin: 0px; + width: 100%; + border-bottom: 1px solid var(--title-separator-color); + background-color: var(--title-background-color); +} + +.image +{ + text-align: center; +} + +.dotgraph +{ + text-align: center; +} + +.mscgraph +{ + text-align: center; +} + +.plantumlgraph +{ + text-align: center; +} + +.diagraph +{ + text-align: center; +} + +.caption +{ + font-weight: bold; +} + +dl.citelist { + margin-bottom:50px; +} + +dl.citelist dt { + color:var(--citation-label-color); + float:left; + font-weight:bold; + margin-right:10px; + padding:5px; + text-align:right; + width:52px; +} + +dl.citelist dd { + margin:2px 0 2px 72px; + padding:5px 0; +} + +div.toc { + padding: 14px 25px; + background-color: var(--toc-background-color); + border: 1px solid var(--toc-border-color); + border-radius: 7px 7px 7px 7px; + float: right; + height: auto; + margin: 0 8px 10px 10px; + width: 200px; +} + +div.toc li { + background: var(--toc-down-arrow-image) no-repeat scroll 0 5px transparent; + font: 10px/1.2 var(--font-family-toc); + margin-top: 5px; + padding-left: 10px; + padding-top: 2px; +} + +div.toc h3 { + font: bold 12px/1.2 var(--font-family-toc); + color: var(--toc-header-color); + border-bottom: 0 none; + margin: 0; +} + +div.toc ul { + list-style: none outside none; + border: medium none; + padding: 0px; +} + +div.toc li.level1 { + margin-left: 0px; +} + +div.toc li.level2 { + margin-left: 15px; +} + +div.toc li.level3 { + margin-left: 15px; +} + +div.toc li.level4 { + margin-left: 15px; +} + +span.emoji { + /* font family used at the site: https://unicode.org/emoji/charts/full-emoji-list.html + * font-family: "Noto Color Emoji", "Apple Color Emoji", "Segoe UI Emoji", Times, Symbola, Aegyptus, Code2000, Code2001, Code2002, Musica, serif, LastResort; + */ +} + +span.obfuscator { + display: none; +} + +.inherit_header { + font-weight: bold; + color: var(--inherit-header-color); + cursor: pointer; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.inherit_header td { + padding: 6px 0px 2px 5px; +} + +.inherit { + display: none; +} + +tr.heading h2 { + margin-top: 12px; + margin-bottom: 4px; +} + +/* tooltip related style info */ + +.ttc { + position: absolute; + display: none; +} + +#powerTip { + cursor: default; + /*white-space: nowrap;*/ + color: var(--tooltip-foreground-color); + background-color: var(--tooltip-background-color); + border: 1px solid var(--tooltip-border-color); + border-radius: 4px 4px 4px 4px; + box-shadow: var(--tooltip-shadow); + display: none; + font-size: smaller; + max-width: 80%; + opacity: 0.9; + padding: 1ex 1em 1em; + position: absolute; + z-index: 2147483647; +} + +#powerTip div.ttdoc { + color: var(--tooltip-doc-color); + font-style: italic; +} + +#powerTip div.ttname a { + font-weight: bold; +} + +#powerTip a { + color: var(--tooltip-link-color); +} + +#powerTip div.ttname { + font-weight: bold; +} + +#powerTip div.ttdeci { + color: var(--tooltip-declaration-color); +} + +#powerTip div { + margin: 0px; + padding: 0px; + font-size: 12px; + font-family: var(--font-family-tooltip); + line-height: 16px; +} + +#powerTip:before, #powerTip:after { + content: ""; + position: absolute; + margin: 0px; +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.s:after, #powerTip.s:before, +#powerTip.w:after, #powerTip.w:before, +#powerTip.e:after, #powerTip.e:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.nw:after, #powerTip.nw:before, +#powerTip.sw:after, #powerTip.sw:before { + border: solid transparent; + content: " "; + height: 0; + width: 0; + position: absolute; +} + +#powerTip.n:after, #powerTip.s:after, +#powerTip.w:after, #powerTip.e:after, +#powerTip.nw:after, #powerTip.ne:after, +#powerTip.sw:after, #powerTip.se:after { + border-color: rgba(255, 255, 255, 0); +} + +#powerTip.n:before, #powerTip.s:before, +#powerTip.w:before, #powerTip.e:before, +#powerTip.nw:before, #powerTip.ne:before, +#powerTip.sw:before, #powerTip.se:before { + border-color: rgba(128, 128, 128, 0); +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.nw:after, #powerTip.nw:before { + top: 100%; +} + +#powerTip.n:after, #powerTip.ne:after, #powerTip.nw:after { + border-top-color: var(--tooltip-background-color); + border-width: 10px; + margin: 0px -10px; +} +#powerTip.n:before, #powerTip.ne:before, #powerTip.nw:before { + border-top-color: var(--tooltip-border-color); + border-width: 11px; + margin: 0px -11px; +} +#powerTip.n:after, #powerTip.n:before { + left: 50%; +} + +#powerTip.nw:after, #powerTip.nw:before { + right: 14px; +} + +#powerTip.ne:after, #powerTip.ne:before { + left: 14px; +} + +#powerTip.s:after, #powerTip.s:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.sw:after, #powerTip.sw:before { + bottom: 100%; +} + +#powerTip.s:after, #powerTip.se:after, #powerTip.sw:after { + border-bottom-color: var(--tooltip-background-color); + border-width: 10px; + margin: 0px -10px; +} + +#powerTip.s:before, #powerTip.se:before, #powerTip.sw:before { + border-bottom-color: var(--tooltip-border-color); + border-width: 11px; + margin: 0px -11px; +} + +#powerTip.s:after, #powerTip.s:before { + left: 50%; +} + +#powerTip.sw:after, #powerTip.sw:before { + right: 14px; +} + +#powerTip.se:after, #powerTip.se:before { + left: 14px; +} + +#powerTip.e:after, #powerTip.e:before { + left: 100%; +} +#powerTip.e:after { + border-left-color: var(--tooltip-border-color); + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.e:before { + border-left-color: var(--tooltip-border-color); + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +#powerTip.w:after, #powerTip.w:before { + right: 100%; +} +#powerTip.w:after { + border-right-color: var(--tooltip-border-color); + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.w:before { + border-right-color: var(--tooltip-border-color); + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +@media print +{ + #top { display: none; } + #side-nav { display: none; } + #nav-path { display: none; } + body { overflow:visible; } + h1, h2, h3, h4, h5, h6 { page-break-after: avoid; } + .summary { display: none; } + .memitem { page-break-inside: avoid; } + #doc-content + { + margin-left:0 !important; + height:auto !important; + width:auto !important; + overflow:inherit; + display:inline; + } +} + +/* @group Markdown */ + +table.markdownTable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.markdownTable td, table.markdownTable th { + border: 1px solid var(--table-cell-border-color); + padding: 3px 7px 2px; +} + +table.markdownTable tr { +} + +th.markdownTableHeadLeft, th.markdownTableHeadRight, th.markdownTableHeadCenter, th.markdownTableHeadNone { + background-color: var(--table-header-background-color); + color: var(--table-header-foreground-color); + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +th.markdownTableHeadLeft, td.markdownTableBodyLeft { + text-align: left +} + +th.markdownTableHeadRight, td.markdownTableBodyRight { + text-align: right +} + +th.markdownTableHeadCenter, td.markdownTableBodyCenter { + text-align: center +} + +tt, code, kbd, samp +{ + display: inline-block; +} +/* @end */ + +u { + text-decoration: underline; +} + +details>summary { + list-style-type: none; +} + +details > summary::-webkit-details-marker { + display: none; +} + +details>summary::before { + content: "\25ba"; + padding-right:4px; + font-size: 80%; +} + +details[open]>summary::before { + content: "\25bc"; + padding-right:4px; + font-size: 80%; +} + +body { + scrollbar-color: var(--scrollbar-thumb-color) var(--scrollbar-background-color); +} + +::-webkit-scrollbar { + background-color: var(--scrollbar-background-color); + height: 12px; + width: 12px; +} +::-webkit-scrollbar-thumb { + border-radius: 6px; + box-shadow: inset 0 0 12px 12px var(--scrollbar-thumb-color); + border: solid 2px transparent; +} +::-webkit-scrollbar-corner { + background-color: var(--scrollbar-background-color); +} + diff --git a/docs/doxygen.svg b/docs/doxygen.svg new file mode 100644 index 0000000000000000000000000000000000000000..79a76354078d0fe34ea30c3a481405c720937483 --- /dev/null +++ b/docs/doxygen.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/dynsections.js b/docs/dynsections.js new file mode 100644 index 0000000000000000000000000000000000000000..f579fbf3e2bb52ac037f1304107e2b43526e08c5 --- /dev/null +++ b/docs/dynsections.js @@ -0,0 +1,123 @@ +/* + @licstart The following is the entire license notice for the JavaScript code in this file. + + The MIT License (MIT) + + Copyright (C) 1997-2020 by Dimitri van Heesch + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + @licend The above is the entire license notice for the JavaScript code in this file + */ +function toggleVisibility(linkObj) +{ + var base = $(linkObj).attr('id'); + var summary = $('#'+base+'-summary'); + var content = $('#'+base+'-content'); + var trigger = $('#'+base+'-trigger'); + var src=$(trigger).attr('src'); + if (content.is(':visible')===true) { + content.hide(); + summary.show(); + $(linkObj).addClass('closed').removeClass('opened'); + $(trigger).attr('src',src.substring(0,src.length-8)+'closed.png'); + } else { + content.show(); + summary.hide(); + $(linkObj).removeClass('closed').addClass('opened'); + $(trigger).attr('src',src.substring(0,src.length-10)+'open.png'); + } + return false; +} + +function updateStripes() +{ + $('table.directory tr'). + removeClass('even').filter(':visible:even').addClass('even'); + $('table.directory tr'). + removeClass('odd').filter(':visible:odd').addClass('odd'); +} + +function toggleLevel(level) +{ + $('table.directory tr').each(function() { + var l = this.id.split('_').length-1; + var i = $('#img'+this.id.substring(3)); + var a = $('#arr'+this.id.substring(3)); + if (l + + + + + + +intel-ipsec-mb: File List + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
File List
+
+
+
Here is a list of all files with brief descriptions:
+ + +
 intel-ipsec-mb.h
+
+
+
+ + + + diff --git a/docs/folderclosed.svg b/docs/folderclosed.svg new file mode 100644 index 0000000000000000000000000000000000000000..b04bed2e723617bea1e7dda93638dddf7e5971ed --- /dev/null +++ b/docs/folderclosed.svg @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/docs/folderclosedd.svg b/docs/folderclosedd.svg new file mode 100644 index 0000000000000000000000000000000000000000..52f0166a23e730c238954892aecd3babfabf75d0 --- /dev/null +++ b/docs/folderclosedd.svg @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/docs/folderopen.svg b/docs/folderopen.svg new file mode 100644 index 0000000000000000000000000000000000000000..f6896dd254b6d25580338d9c9a097fc46e445355 --- /dev/null +++ b/docs/folderopen.svg @@ -0,0 +1,17 @@ + + + + + + + + + + diff --git a/docs/folderopend.svg b/docs/folderopend.svg new file mode 100644 index 0000000000000000000000000000000000000000..2d1f06e7bc6e85be079b7ee1346c79baa4c50ce1 --- /dev/null +++ b/docs/folderopend.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + diff --git a/docs/functions.html b/docs/functions.html new file mode 100644 index 0000000000000000000000000000000000000000..c99b9a060fe939edb0f9f1b002038a90e46faf5c --- /dev/null +++ b/docs/functions.html @@ -0,0 +1,124 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- _ -

+
+
+ + + + diff --git a/docs/functions_a.html b/docs/functions_a.html new file mode 100644 index 0000000000000000000000000000000000000000..70bc53dd59e927410a5c18cc2bca3da930d78a63 --- /dev/null +++ b/docs/functions_a.html @@ -0,0 +1,133 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- a -

+
+
+ + + + diff --git a/docs/functions_c.html b/docs/functions_c.html new file mode 100644 index 0000000000000000000000000000000000000000..5c9d313c09eb67fedb9b294fc37f976ecf1fe604 --- /dev/null +++ b/docs/functions_c.html @@ -0,0 +1,144 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- c -

+
+
+ + + + diff --git a/docs/functions_d.html b/docs/functions_d.html new file mode 100644 index 0000000000000000000000000000000000000000..4d5c84def3106572917990e48d270d0a39ad3072 --- /dev/null +++ b/docs/functions_d.html @@ -0,0 +1,125 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- d -

+
+
+ + + + diff --git a/docs/functions_e.html b/docs/functions_e.html new file mode 100644 index 0000000000000000000000000000000000000000..9d5b2f53880fa78f109fa3ec24977740a5cbaafb --- /dev/null +++ b/docs/functions_e.html @@ -0,0 +1,121 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- e -

+
+
+ + + + diff --git a/docs/functions_f.html b/docs/functions_f.html new file mode 100644 index 0000000000000000000000000000000000000000..d7079ea3c6a1de42335bc7be8f272c41722295d7 --- /dev/null +++ b/docs/functions_f.html @@ -0,0 +1,124 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- f -

+
+
+ + + + diff --git a/docs/functions_g.html b/docs/functions_g.html new file mode 100644 index 0000000000000000000000000000000000000000..63178cf664867928e60bf418ab3d3408772c330c --- /dev/null +++ b/docs/functions_g.html @@ -0,0 +1,160 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- g -

+
+
+ + + + diff --git a/docs/functions_h.html b/docs/functions_h.html new file mode 100644 index 0000000000000000000000000000000000000000..c45c42c1b1c9b78b212bb6df0cbf5412081e2ad2 --- /dev/null +++ b/docs/functions_h.html @@ -0,0 +1,127 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- h -

+
+
+ + + + diff --git a/docs/functions_i.html b/docs/functions_i.html new file mode 100644 index 0000000000000000000000000000000000000000..66964a82a8961945c8a27024c316a5d5f5eb53b4 --- /dev/null +++ b/docs/functions_i.html @@ -0,0 +1,118 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- i -

+
+
+ + + + diff --git a/docs/functions_j.html b/docs/functions_j.html new file mode 100644 index 0000000000000000000000000000000000000000..5fe32b241472a1bd0584f0fca05f3ed64c08b915 --- /dev/null +++ b/docs/functions_j.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- j -

+
+
+ + + + diff --git a/docs/functions_k.html b/docs/functions_k.html new file mode 100644 index 0000000000000000000000000000000000000000..7f136321162c7ac6b46453a0fcda028c1f5fbdc3 --- /dev/null +++ b/docs/functions_k.html @@ -0,0 +1,121 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- k -

+
+
+ + + + diff --git a/docs/functions_l.html b/docs/functions_l.html new file mode 100644 index 0000000000000000000000000000000000000000..01dc81b61fe7f0dbf118ee706b1f553729f1bb2b --- /dev/null +++ b/docs/functions_l.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- l -

+
+
+ + + + diff --git a/docs/functions_m.html b/docs/functions_m.html new file mode 100644 index 0000000000000000000000000000000000000000..6cc0ff4516c432192017cc2ea2abe8fe0e02a682 --- /dev/null +++ b/docs/functions_m.html @@ -0,0 +1,118 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- m -

+
+
+ + + + diff --git a/docs/functions_n.html b/docs/functions_n.html new file mode 100644 index 0000000000000000000000000000000000000000..55f3311effd0dd0514a2e38b4e8447953708a2a3 --- /dev/null +++ b/docs/functions_n.html @@ -0,0 +1,115 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- n -

+
+
+ + + + diff --git a/docs/functions_o.html b/docs/functions_o.html new file mode 100644 index 0000000000000000000000000000000000000000..f79290c5552f33f6174c8d1c0be356304fb516d4 --- /dev/null +++ b/docs/functions_o.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- o -

+
+
+ + + + diff --git a/docs/functions_p.html b/docs/functions_p.html new file mode 100644 index 0000000000000000000000000000000000000000..2a1a4d50dc6705233e59d7e1fdeb06c86aa0d8a3 --- /dev/null +++ b/docs/functions_p.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- p -

+
+
+ + + + diff --git a/docs/functions_q.html b/docs/functions_q.html new file mode 100644 index 0000000000000000000000000000000000000000..b7d8c2c95664f53b9e0010cb61ff7699d284c245 --- /dev/null +++ b/docs/functions_q.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- q -

+
+
+ + + + diff --git a/docs/functions_r.html b/docs/functions_r.html new file mode 100644 index 0000000000000000000000000000000000000000..148252a204cf23cd35f7d2df2b9ed644693acd0e --- /dev/null +++ b/docs/functions_r.html @@ -0,0 +1,115 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- r -

+
+
+ + + + diff --git a/docs/functions_s.html b/docs/functions_s.html new file mode 100644 index 0000000000000000000000000000000000000000..5fe909e240a72b57ebfe1bc363d593edbe859c0f --- /dev/null +++ b/docs/functions_s.html @@ -0,0 +1,161 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- s -

+
+
+ + + + diff --git a/docs/functions_u.html b/docs/functions_u.html new file mode 100644 index 0000000000000000000000000000000000000000..4b198c88ecd0cb35d3d4cba1910a5d8a1cd8fe1c --- /dev/null +++ b/docs/functions_u.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- u -

+
+
+ + + + diff --git a/docs/functions_v.html b/docs/functions_v.html new file mode 100644 index 0000000000000000000000000000000000000000..f0f9122cc729403e0ae3a8c09bc3a5f4d472dd14 --- /dev/null +++ b/docs/functions_v.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- v -

+
+
+ + + + diff --git a/docs/functions_vars.html b/docs/functions_vars.html new file mode 100644 index 0000000000000000000000000000000000000000..482e041faed09ae15a49806951a99faa1675882e --- /dev/null +++ b/docs/functions_vars.html @@ -0,0 +1,124 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- _ -

+
+
+ + + + diff --git a/docs/functions_vars_a.html b/docs/functions_vars_a.html new file mode 100644 index 0000000000000000000000000000000000000000..e45e43e8d7b5caf5267c2bae8d70a2343ee7c50d --- /dev/null +++ b/docs/functions_vars_a.html @@ -0,0 +1,133 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- a -

+
+
+ + + + diff --git a/docs/functions_vars_c.html b/docs/functions_vars_c.html new file mode 100644 index 0000000000000000000000000000000000000000..ffa1535609f5830d0e384123f5b9db64ac3bd5d3 --- /dev/null +++ b/docs/functions_vars_c.html @@ -0,0 +1,144 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- c -

+
+
+ + + + diff --git a/docs/functions_vars_d.html b/docs/functions_vars_d.html new file mode 100644 index 0000000000000000000000000000000000000000..4a6c5347a0b470a75a508bc1eabfe189c513837b --- /dev/null +++ b/docs/functions_vars_d.html @@ -0,0 +1,125 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- d -

+
+
+ + + + diff --git a/docs/functions_vars_e.html b/docs/functions_vars_e.html new file mode 100644 index 0000000000000000000000000000000000000000..6ccb1419a10ac0064ddaac301f4bd2f24e839ec5 --- /dev/null +++ b/docs/functions_vars_e.html @@ -0,0 +1,121 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- e -

+
+
+ + + + diff --git a/docs/functions_vars_f.html b/docs/functions_vars_f.html new file mode 100644 index 0000000000000000000000000000000000000000..522db76a77206f60158e45c7d58cb3057b8b4eb1 --- /dev/null +++ b/docs/functions_vars_f.html @@ -0,0 +1,124 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- f -

+
+
+ + + + diff --git a/docs/functions_vars_g.html b/docs/functions_vars_g.html new file mode 100644 index 0000000000000000000000000000000000000000..b5c5ea5729cda0e1a35a4dc373e09baddf4b743e --- /dev/null +++ b/docs/functions_vars_g.html @@ -0,0 +1,160 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- g -

+
+
+ + + + diff --git a/docs/functions_vars_h.html b/docs/functions_vars_h.html new file mode 100644 index 0000000000000000000000000000000000000000..5718002e8cd628c8daff30204cca3737588d83d0 --- /dev/null +++ b/docs/functions_vars_h.html @@ -0,0 +1,127 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- h -

+
+
+ + + + diff --git a/docs/functions_vars_i.html b/docs/functions_vars_i.html new file mode 100644 index 0000000000000000000000000000000000000000..cfa665bd9c71a50534c707059f0bbe3e4526c422 --- /dev/null +++ b/docs/functions_vars_i.html @@ -0,0 +1,118 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- i -

+
+
+ + + + diff --git a/docs/functions_vars_j.html b/docs/functions_vars_j.html new file mode 100644 index 0000000000000000000000000000000000000000..cacd691aadeec97657bef6bb545e3c0731e74687 --- /dev/null +++ b/docs/functions_vars_j.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- j -

+
+
+ + + + diff --git a/docs/functions_vars_k.html b/docs/functions_vars_k.html new file mode 100644 index 0000000000000000000000000000000000000000..9b748c540456e0e4faac9ebc2473da20e69be87b --- /dev/null +++ b/docs/functions_vars_k.html @@ -0,0 +1,121 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- k -

+
+
+ + + + diff --git a/docs/functions_vars_l.html b/docs/functions_vars_l.html new file mode 100644 index 0000000000000000000000000000000000000000..f44a88428b5998cef89897d21484e1a037e9a233 --- /dev/null +++ b/docs/functions_vars_l.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- l -

+
+
+ + + + diff --git a/docs/functions_vars_m.html b/docs/functions_vars_m.html new file mode 100644 index 0000000000000000000000000000000000000000..e1db0db74619b417d438b39a8f319826e8ca8db5 --- /dev/null +++ b/docs/functions_vars_m.html @@ -0,0 +1,118 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- m -

+
+
+ + + + diff --git a/docs/functions_vars_n.html b/docs/functions_vars_n.html new file mode 100644 index 0000000000000000000000000000000000000000..3d36b8398fb3d71944af5af6ee6440cb38e6bccd --- /dev/null +++ b/docs/functions_vars_n.html @@ -0,0 +1,115 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- n -

+
+
+ + + + diff --git a/docs/functions_vars_o.html b/docs/functions_vars_o.html new file mode 100644 index 0000000000000000000000000000000000000000..c91ed705ee866fba3bb07feaf3297df914bd35e4 --- /dev/null +++ b/docs/functions_vars_o.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- o -

+
+
+ + + + diff --git a/docs/functions_vars_p.html b/docs/functions_vars_p.html new file mode 100644 index 0000000000000000000000000000000000000000..3b6a4275e703b66c590e418ef072dc4ad85ccd0f --- /dev/null +++ b/docs/functions_vars_p.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- p -

+
+
+ + + + diff --git a/docs/functions_vars_q.html b/docs/functions_vars_q.html new file mode 100644 index 0000000000000000000000000000000000000000..2daf974b529a83f3a1d36f9a5d7627fdca87cc67 --- /dev/null +++ b/docs/functions_vars_q.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- q -

+
+
+ + + + diff --git a/docs/functions_vars_r.html b/docs/functions_vars_r.html new file mode 100644 index 0000000000000000000000000000000000000000..89cb6fb335b004b63d2e10337f7e9d3acb8f463c --- /dev/null +++ b/docs/functions_vars_r.html @@ -0,0 +1,115 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- r -

+
+
+ + + + diff --git a/docs/functions_vars_s.html b/docs/functions_vars_s.html new file mode 100644 index 0000000000000000000000000000000000000000..e71248271808892253b91f645706a8be208c3fc3 --- /dev/null +++ b/docs/functions_vars_s.html @@ -0,0 +1,161 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- s -

+
+
+ + + + diff --git a/docs/functions_vars_u.html b/docs/functions_vars_u.html new file mode 100644 index 0000000000000000000000000000000000000000..cd6c676af8ab565a2fde220ab634bf3396dc1047 --- /dev/null +++ b/docs/functions_vars_u.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- u -

+
+
+ + + + diff --git a/docs/functions_vars_v.html b/docs/functions_vars_v.html new file mode 100644 index 0000000000000000000000000000000000000000..48f28efbbe28bed5f737adba7b99df1e71b9bc6e --- /dev/null +++ b/docs/functions_vars_v.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- v -

+
+
+ + + + diff --git a/docs/functions_vars_x.html b/docs/functions_vars_x.html new file mode 100644 index 0000000000000000000000000000000000000000..278945ea5dfade771132cb4cb2ca875d441087d7 --- /dev/null +++ b/docs/functions_vars_x.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- x -

+
+
+ + + + diff --git a/docs/functions_vars_z.html b/docs/functions_vars_z.html new file mode 100644 index 0000000000000000000000000000000000000000..370b0b31bbb7e24df26ec63eb02ebdbf80c0e442 --- /dev/null +++ b/docs/functions_vars_z.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all variables with links to the structures/unions they belong to:
+ +

- z -

+
+
+ + + + diff --git a/docs/functions_x.html b/docs/functions_x.html new file mode 100644 index 0000000000000000000000000000000000000000..668dda1273898e48824b6afc98fe15f513256721 --- /dev/null +++ b/docs/functions_x.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- x -

+
+
+ + + + diff --git a/docs/functions_z.html b/docs/functions_z.html new file mode 100644 index 0000000000000000000000000000000000000000..bf6dd4058584294ff2d7bdb719c52bf864126e66 --- /dev/null +++ b/docs/functions_z.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- z -

+
+
+ + + + diff --git a/docs/globals.html b/docs/globals.html new file mode 100644 index 0000000000000000000000000000000000000000..b64eb346f76153aed4afba5a309fdfe9d72e557a --- /dev/null +++ b/docs/globals.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- _ -

+
+
+ + + + diff --git a/docs/globals_a.html b/docs/globals_a.html new file mode 100644 index 0000000000000000000000000000000000000000..756b51e2b8000ec8213efad786c06be6ff1b875d --- /dev/null +++ b/docs/globals_a.html @@ -0,0 +1,239 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- a -

+
+
+ + + + diff --git a/docs/globals_b.html b/docs/globals_b.html new file mode 100644 index 0000000000000000000000000000000000000000..44a6f7dedf53c4ae270092b551db74a31103abda --- /dev/null +++ b/docs/globals_b.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- b -

+
+
+ + + + diff --git a/docs/globals_c.html b/docs/globals_c.html new file mode 100644 index 0000000000000000000000000000000000000000..070ba8f29e845515a95d63d824b278044ed72011 --- /dev/null +++ b/docs/globals_c.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- c -

+
+
+ + + + diff --git a/docs/globals_d.html b/docs/globals_d.html new file mode 100644 index 0000000000000000000000000000000000000000..840247cdd4075dc3e053531d9d20676ff18e04ec --- /dev/null +++ b/docs/globals_d.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- d -

+
+
+ + + + diff --git a/docs/globals_defs.html b/docs/globals_defs.html new file mode 100644 index 0000000000000000000000000000000000000000..8b6e237d8f0daa4baf27233e9c86ddf12ff43d9f --- /dev/null +++ b/docs/globals_defs.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all macros with links to the files they belong to:
+ +

- _ -

+
+
+ + + + diff --git a/docs/globals_defs.js b/docs/globals_defs.js new file mode 100644 index 0000000000000000000000000000000000000000..a832e02c7daa73ca50dc922137e9eb4439d5887a --- /dev/null +++ b/docs/globals_defs.js @@ -0,0 +1,7 @@ +var globals_defs = +[ + [ "_", "globals_defs.html", null ], + [ "d", "globals_defs_d.html", null ], + [ "i", "globals_defs_i.html", null ], + [ "k", "globals_defs_k.html", null ] +]; \ No newline at end of file diff --git a/docs/globals_defs_d.html b/docs/globals_defs_d.html new file mode 100644 index 0000000000000000000000000000000000000000..f46d9403cd7119cc9011929f29ea01b6215d5a7d --- /dev/null +++ b/docs/globals_defs_d.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all macros with links to the files they belong to:
+ +

- d -

+
+
+ + + + diff --git a/docs/globals_defs_i.html b/docs/globals_defs_i.html new file mode 100644 index 0000000000000000000000000000000000000000..fbae9ceb350468bb7a2e57c832ab6deef77ab3fe --- /dev/null +++ b/docs/globals_defs_i.html @@ -0,0 +1,317 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all macros with links to the files they belong to:
+ +

- i -

+
+
+ + + + diff --git a/docs/globals_defs_k.html b/docs/globals_defs_k.html new file mode 100644 index 0000000000000000000000000000000000000000..608dd3fc6bb909faa1d9f1a12c4ec10b20dc50df --- /dev/null +++ b/docs/globals_defs_k.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all macros with links to the files they belong to:
+ +

- k -

+
+
+ + + + diff --git a/docs/globals_dup.js b/docs/globals_dup.js new file mode 100644 index 0000000000000000000000000000000000000000..2c6350d1ce74ea03b9eac1fe2ee2ab8929b936e7 --- /dev/null +++ b/docs/globals_dup.js @@ -0,0 +1,18 @@ +var globals_dup = +[ + [ "_", "globals.html", null ], + [ "a", "globals_a.html", null ], + [ "b", "globals_b.html", null ], + [ "c", "globals_c.html", null ], + [ "d", "globals_d.html", null ], + [ "f", "globals_f.html", null ], + [ "g", "globals_g.html", null ], + [ "h", "globals_h.html", null ], + [ "i", "globals_i.html", null ], + [ "k", "globals_k.html", null ], + [ "m", "globals_m.html", null ], + [ "q", "globals_q.html", null ], + [ "s", "globals_s.html", null ], + [ "x", "globals_x.html", null ], + [ "z", "globals_z.html", null ] +]; \ No newline at end of file diff --git a/docs/globals_enum.html b/docs/globals_enum.html new file mode 100644 index 0000000000000000000000000000000000000000..1d9bd5d977a352801082020a61c22e09174c4800 --- /dev/null +++ b/docs/globals_enum.html @@ -0,0 +1,119 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all enums with links to the files they belong to:
+
+
+ + + + diff --git a/docs/globals_eval.html b/docs/globals_eval.html new file mode 100644 index 0000000000000000000000000000000000000000..929f94b131175aa9bdba77cd900c274ac6661587 --- /dev/null +++ b/docs/globals_eval.html @@ -0,0 +1,264 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all enum values with links to the files they belong to:
+ +

- i -

+
+
+ + + + diff --git a/docs/globals_f.html b/docs/globals_f.html new file mode 100644 index 0000000000000000000000000000000000000000..8f30662f17f6f77ec9e814c49d6a4c205c314025 --- /dev/null +++ b/docs/globals_f.html @@ -0,0 +1,118 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- f -

+
+
+ + + + diff --git a/docs/globals_func.html b/docs/globals_func.html new file mode 100644 index 0000000000000000000000000000000000000000..0330c1c4c286a391362819900822a0e18cdc2737 --- /dev/null +++ b/docs/globals_func.html @@ -0,0 +1,316 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions with links to the files they belong to:
+ +

- a -

+ + +

- d -

+ + +

- f -

+ + +

- g -

+ + +

- i -

+ + +

- k -

+ + +

- m -

+ + +

- q -

+ + +

- s -

+ + +

- z -

+
+
+ + + + diff --git a/docs/globals_g.html b/docs/globals_g.html new file mode 100644 index 0000000000000000000000000000000000000000..d35c5f4476f421274a655c178c83adc8718e1f4d --- /dev/null +++ b/docs/globals_g.html @@ -0,0 +1,123 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- g -

+
+
+ + + + diff --git a/docs/globals_h.html b/docs/globals_h.html new file mode 100644 index 0000000000000000000000000000000000000000..c4925c0544eebfc20c59c007f50c4c08e461e7b8 --- /dev/null +++ b/docs/globals_h.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- h -

+
+
+ + + + diff --git a/docs/globals_i.html b/docs/globals_i.html new file mode 100644 index 0000000000000000000000000000000000000000..3e4978b27729aed75bd793678dd644e027761e23 --- /dev/null +++ b/docs/globals_i.html @@ -0,0 +1,498 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- i -

+
+
+ + + + diff --git a/docs/globals_k.html b/docs/globals_k.html new file mode 100644 index 0000000000000000000000000000000000000000..10d20f6b450302e512a5953c90b149821af221cf --- /dev/null +++ b/docs/globals_k.html @@ -0,0 +1,128 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- k -

+
+
+ + + + diff --git a/docs/globals_m.html b/docs/globals_m.html new file mode 100644 index 0000000000000000000000000000000000000000..a821a2d8c7f894316061d4f147493be20c3b2ac2 --- /dev/null +++ b/docs/globals_m.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- m -

+
+
+ + + + diff --git a/docs/globals_q.html b/docs/globals_q.html new file mode 100644 index 0000000000000000000000000000000000000000..891e068a39783c361dd697b0b954d9c3d1845d0b --- /dev/null +++ b/docs/globals_q.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- q -

+
+
+ + + + diff --git a/docs/globals_s.html b/docs/globals_s.html new file mode 100644 index 0000000000000000000000000000000000000000..1ee3820823f66c4779c66f3fbdb8f650bc663208 --- /dev/null +++ b/docs/globals_s.html @@ -0,0 +1,137 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- s -

+
+
+ + + + diff --git a/docs/globals_type.html b/docs/globals_type.html new file mode 100644 index 0000000000000000000000000000000000000000..d8be48ae13c39c7b4d240f0a118c942c265c49b7 --- /dev/null +++ b/docs/globals_type.html @@ -0,0 +1,226 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all typedefs with links to the files they belong to:
+ +

- a -

+ + +

- b -

+ + +

- c -

+ + +

- d -

+ + +

- f -

+ + +

- g -

+ + +

- h -

+ + +

- i -

+ + +

- k -

+ + +

- q -

+ + +

- s -

+ + +

- x -

+ + +

- z -

+
+
+ + + + diff --git a/docs/globals_x.html b/docs/globals_x.html new file mode 100644 index 0000000000000000000000000000000000000000..7563e1710ae1ade0673aba85e8c2344bf60aebf2 --- /dev/null +++ b/docs/globals_x.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- x -

+
+
+ + + + diff --git a/docs/globals_z.html b/docs/globals_z.html new file mode 100644 index 0000000000000000000000000000000000000000..e86800937072d6599a5523cc5da104fdc77e1426 --- /dev/null +++ b/docs/globals_z.html @@ -0,0 +1,119 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- z -

+
+
+ + + + diff --git a/docs/intel-ipsec-mb_8h.html b/docs/intel-ipsec-mb_8h.html new file mode 100644 index 0000000000000000000000000000000000000000..73fdf9b9ca7ecdf71143ad0850651619c3ef756b --- /dev/null +++ b/docs/intel-ipsec-mb_8h.html @@ -0,0 +1,18640 @@ + + + + + + + +intel-ipsec-mb: intel-ipsec-mb.h File Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Structures | +Macros | +Typedefs | +Enumerations | +Functions
+
intel-ipsec-mb.h File Reference
+
+
+
#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
+
+

Go to the source code of this file.

+ + + + + + + + + + + + + + + + + + + + + + + +

+Data Structures

struct  imb_uint128_t
 
struct  IMB_SGL_IOV
 
struct  IMB_JOB
 
struct  kasumi_key_sched_s
 
struct  gcm_context_data
 holds GCM operation context More...
 
struct  chacha20_poly1305_context_data
 holds Chacha20-Poly1305 operation context More...
 
struct  gcm_key_data
 holds intermediate key data needed to improve performance More...
 
struct  snow3g_key_schedule_s
 
struct  IMB_MGR
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Macros

#define DECLARE_ALIGNED(decl, alignval)    __declspec(align(alignval)) decl
 
#define __forceinline    static __forceinline
 
#define IMB_DLL_EXPORT
 
#define IMB_DLL_LOCAL
 
#define IMB_VERSION_STR   "1.4.0-dev"
 
#define IMB_VERSION_NUM   0x10400
 
#define IMB_VERSION(a, b, c)   (((a) << 16) + ((b) << 8) + (c))
 
#define IMB_ASSERT(x)
 
#define IMB_DIM(x)   (sizeof(x) / sizeof(x[0]))
 
#define IMB_DES_KEY_SCHED_SIZE   (16 * 8)
 
#define IMB_DES_BLOCK_SIZE   8
 
#define IMB_AES_BLOCK_SIZE   16
 
#define IMB_SHA1_DIGEST_SIZE_IN_BYTES   20
 
#define IMB_SHA224_DIGEST_SIZE_IN_BYTES   28
 
#define IMB_SHA256_DIGEST_SIZE_IN_BYTES   32
 
#define IMB_SHA384_DIGEST_SIZE_IN_BYTES   48
 
#define IMB_SHA512_DIGEST_SIZE_IN_BYTES   64
 
#define IMB_MD5_DIGEST_SIZE_IN_BYTES   16
 
#define IMB_SHA1_BLOCK_SIZE   64
 
#define IMB_SHA_224_BLOCK_SIZE   64
 
#define IMB_SHA_256_BLOCK_SIZE   64
 
#define IMB_SHA_384_BLOCK_SIZE   128
 
#define IMB_SHA_512_BLOCK_SIZE   128
 
#define IMB_MD5_BLOCK_SIZE   64
 
#define IMB_KASUMI_KEY_SIZE   16
 
#define IMB_KASUMI_IV_SIZE   8
 
#define IMB_KASUMI_BLOCK_SIZE   8
 
#define IMB_KASUMI_DIGEST_SIZE   4
 
#define IMB_ZUC_KEY_LEN_IN_BYTES   16
 
#define IMB_ZUC_IV_LEN_IN_BYTES   16
 
#define IMB_ZUC256_KEY_LEN_IN_BYTES   32
 
#define IMB_ZUC256_IV_LEN_IN_BYTES_MIN   23
 
#define IMB_ZUC256_IV_LEN_IN_BYTES_MAX   25
 
#define IMB_ZUC_DIGEST_LEN_IN_BYTES   4
 
#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN   4
 
#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX   16
 
#define IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE   14
 
#define IMB_DOCSIS_CRC32_TAG_SIZE   4
 
#define KASUMI_KEY_SCHEDULE_SIZE   64
 
#define IMB_GCM_BLOCK_LEN   16
 
#define IMB_MAX_TAG_LEN   (16)
 
#define IMB_GCM_IV_DATA_LEN   (12)
 
#define IMB_GCM_128_KEY_LEN   (16)
 
#define IMB_GCM_192_KEY_LEN   (24)
 
#define IMB_GCM_256_KEY_LEN   (32)
 
#define IMB_GCM_ENC_KEY_LEN   16
 
#define IMB_GCM_KEY_SETS   (15)
 
#define IMB_FLAG_SHANI_OFF   (1ULL << 0)
 
#define IMB_FLAG_AESNI_OFF   (1ULL << 1)
 
#define IMB_FLAG_GFNI_OFF   (1ULL << 2)
 
#define IMB_FEATURE_SHANI   (1ULL << 0)
 
#define IMB_FEATURE_AESNI   (1ULL << 1)
 
#define IMB_FEATURE_PCLMULQDQ   (1ULL << 2)
 
#define IMB_FEATURE_CMOV   (1ULL << 3)
 
#define IMB_FEATURE_SSE4_2   (1ULL << 4)
 
#define IMB_FEATURE_AVX   (1ULL << 5)
 
#define IMB_FEATURE_AVX2   (1ULL << 6)
 
#define IMB_FEATURE_AVX512F   (1ULL << 7)
 
#define IMB_FEATURE_AVX512DQ   (1ULL << 8)
 
#define IMB_FEATURE_AVX512CD   (1ULL << 9)
 
#define IMB_FEATURE_AVX512BW   (1ULL << 10)
 
#define IMB_FEATURE_AVX512VL   (1ULL << 11)
 
#define IMB_FEATURE_AVX512_SKX
 
#define IMB_FEATURE_VAES   (1ULL << 12)
 
#define IMB_FEATURE_VPCLMULQDQ   (1ULL << 13)
 
#define IMB_FEATURE_SAFE_DATA   (1ULL << 14)
 
#define IMB_FEATURE_SAFE_PARAM   (1ULL << 15)
 
#define IMB_FEATURE_GFNI   (1ULL << 16)
 
#define IMB_FEATURE_AVX512_IFMA   (1ULL << 17)
 
#define IMB_FEATURE_BMI2   (1ULL << 18)
 
#define IMB_FEATURE_AESNI_EMU   (1ULL << 19)
 
#define IMB_FEATURE_SELF_TEST   (1ULL << 20) /* self-test feature present */
 
#define IMB_FEATURE_SELF_TEST_PASS   (1ULL << 21) /* self-test passed */
 
#define IMB_FEATURE_AVX_IFMA   (1ULL << 22)
 
#define IMB_CPUFLAGS_NO_AESNI   (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV)
 
#define IMB_CPUFLAGS_SSE
 
#define IMB_CPUFLAGS_SSE_T2   (IMB_CPUFLAGS_SSE | IMB_FEATURE_SHANI)
 
#define IMB_CPUFLAGS_SSE_T3   (IMB_CPUFLAGS_SSE_T2 | IMB_FEATURE_GFNI)
 
#define IMB_CPUFLAGS_AVX   (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX)
 
#define IMB_CPUFLAGS_AVX2
 
#define IMB_CPUFLAGS_AVX512   (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX)
 
#define IMB_CPUFLAGS_AVX512_T2
 
#define IMB_CPUFLAGS_AVX2_T2
 
#define IMB_CPUFLAGS_AVX_T2
 
#define IMB_CPUFLAGS_AVX2_T3   (IMB_CPUFLAGS_AVX2_T2 | IMB_FEATURE_AVX_IFMA)
 
#define IMB_MAX_BURST_SIZE   128
 
#define IMB_MAX_JOBS   (IMB_MAX_BURST_SIZE * 2)
 
#define IMB_GET_NEXT_JOB(_mgr)   ((_mgr)->get_next_job((_mgr)))
 Get next available job.
 
#define IMB_SUBMIT_JOB(_mgr)   ((_mgr)->submit_job((_mgr)))
 Submit job for processing after validating.
 
#define IMB_SUBMIT_JOB_NOCHECK(_mgr)   ((_mgr)->submit_job_nocheck((_mgr)))
 Submit job for processing without validating.
 
#define IMB_GET_COMPLETED_JOB(_mgr)   ((_mgr)->get_completed_job((_mgr)))
 Get next completed job.
 
#define IMB_FLUSH_JOB(_mgr)   ((_mgr)->flush_job((_mgr)))
 Force processing until next job in queue is completed.
 
#define IMB_QUEUE_SIZE(_mgr)   ((_mgr)->queue_size((_mgr)))
 Get number of jobs queued to be processed.
 
#define IMB_GET_NEXT_BURST(_mgr, _n_jobs, _jobs)    ((_mgr)->get_next_burst((_mgr), (_n_jobs), (_jobs)))
 Get next available burst (list of pointers to available IMB_JOB structures).
 
#define IMB_SUBMIT_BURST(_mgr, _n_jobs, _jobs)    ((_mgr)->submit_burst((_mgr), (_n_jobs), (_jobs)))
 Submit multiple jobs to be processed after validating.
 
#define IMB_SUBMIT_BURST_NOCHECK(_mgr, _n_jobs, _jobs)    ((_mgr)->submit_burst_nocheck((_mgr), (_n_jobs), (_jobs)))
 Submit multiple jobs to be processed without validating.
 
#define IMB_FLUSH_BURST(_mgr, _max_jobs, _jobs)    ((_mgr)->flush_burst((_mgr), (_max_jobs), (_jobs)))
 Force up to max_jobs outstanding jobs to completion.
 
#define IMB_SUBMIT_CIPHER_BURST(_mgr, _jobs, _n_jobs, _cipher, _dir, _key_size)
 
#define IMB_SUBMIT_CIPHER_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _cipher, _dir, _key_size)
 
#define IMB_SUBMIT_HASH_BURST(_mgr, _jobs, _n_jobs, _hash)    ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash)))
 
#define IMB_SUBMIT_HASH_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _hash)    ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash)))
 
#define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key)    ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key)))
 
#define IMB_AES_KEYEXP_192(_mgr, _key, _enc_exp_key, _dec_exp_key)    ((_mgr)->keyexp_192((_key), (_enc_exp_key), (_dec_exp_key)))
 
#define IMB_AES_KEYEXP_256(_mgr, _key, _enc_exp_key, _dec_exp_key)    ((_mgr)->keyexp_256((_key), (_enc_exp_key), (_dec_exp_key)))
 
#define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _exp_key, _key1, _key2)    ((_mgr)->cmac_subkey_gen_128((_exp_key), (_key1), (_key2)))
 
#define IMB_AES_CMAC_SUBKEY_GEN_256(_mgr, _exp_key, _key1, _key2)    ((_mgr)->cmac_subkey_gen_256((_exp_key), (_key1), (_key2)))
 
#define IMB_AES_XCBC_KEYEXP(_mgr, _key, _exp_key, _exp_key2, _exp_key3)    ((_mgr)->xcbc_keyexp((_key), (_exp_key), (_exp_key2), (_exp_key3)))
 
#define IMB_DES_KEYSCHED(_mgr, _exp_key, _key)    ((_mgr)->des_key_sched((_exp_key), (_key)))
 
#define IMB_SHA1_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha1_one_block((_src), (_tag)))
 
#define IMB_SHA1(_mgr, _src, _length, _tag)    ((_mgr)->sha1((_src), (_length), (_tag)))
 
#define IMB_SHA224_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha224_one_block((_src), (_tag)))
 
#define IMB_SHA224(_mgr, _src, _length, _tag)    ((_mgr)->sha224((_src), (_length), (_tag)))
 
#define IMB_SHA256_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha256_one_block((_src), (_tag)))
 
#define IMB_SHA256(_mgr, _src, _length, _tag)    ((_mgr)->sha256((_src), (_length), (_tag)))
 
#define IMB_SHA384_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha384_one_block((_src), (_tag)))
 
#define IMB_SHA384(_mgr, _src, _length, _tag)    ((_mgr)->sha384((_src), (_length), (_tag)))
 
#define IMB_SHA512_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha512_one_block((_src), (_tag)))
 
#define IMB_SHA512(_mgr, _src, _length, _tag)    ((_mgr)->sha512((_src), (_length), (_tag)))
 
#define IMB_MD5_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->md5_one_block((_src), (_tag)))
 
#define IMB_AES128_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len)    ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
 AES-CFB-128 Encrypt/Decrypt up to one block.
 
#define IMB_AES256_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len)    ((_mgr)->aes256_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
 AES-CFB-256 Encrypt/Decrypt up to one block.
 
#define IMB_AES128_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES192_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES256_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES128_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES192_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES256_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES128_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl)    ((_mgr)->gcm128_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
 
#define IMB_AES192_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl)    ((_mgr)->gcm192_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
 
#define IMB_AES256_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl)    ((_mgr)->gcm256_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
 
#define IMB_AES128_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, _aadl)
 
#define IMB_AES192_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, _aadl)
 
#define IMB_AES256_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, _aadl)
 
#define IMB_AES128_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm128_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES192_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm192_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES256_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm256_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES128_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm128_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES192_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm192_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES256_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm256_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES128_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm128_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES192_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm192_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES256_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm256_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES128_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm128_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES192_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm192_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES256_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm256_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES128_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl)    ((_mgr)->gmac128_init((_exp_key), (_ctx), (_iv), (_ivl)))
 
#define IMB_AES192_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl)    ((_mgr)->gmac192_init((_exp_key), (_ctx), (_iv), (_ivl)))
 
#define IMB_AES256_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl)    ((_mgr)->gmac256_init((_exp_key), (_ctx), (_iv), (_ivl)))
 
#define IMB_AES128_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len)    ((_mgr)->gmac128_update((_exp_key), (_ctx), (_src), (_len)))
 
#define IMB_AES192_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len)    ((_mgr)->gmac192_update((_exp_key), (_ctx), (_src), (_len)))
 
#define IMB_AES256_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len)    ((_mgr)->gmac256_update((_exp_key), (_ctx), (_src), (_len)))
 
#define IMB_AES128_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gmac128_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES192_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gmac192_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES256_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gmac256_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES128_GCM_PRECOMP(_mgr, _key)    ((_mgr)->gcm128_precomp((_key)))
 
#define IMB_AES192_GCM_PRECOMP(_mgr, _key)    ((_mgr)->gcm192_precomp((_key)))
 
#define IMB_AES256_GCM_PRECOMP(_mgr, _key)    ((_mgr)->gcm256_precomp((_key)))
 
#define IMB_AES128_GCM_PRE(_mgr, _key, _exp_key)    ((_mgr)->gcm128_pre((_key), (_exp_key)))
 
#define IMB_AES192_GCM_PRE(_mgr, _key, _exp_key)    ((_mgr)->gcm192_pre((_key), (_exp_key)))
 
#define IMB_AES256_GCM_PRE(_mgr, _key, _exp_key)    ((_mgr)->gcm256_pre((_key), (_exp_key)))
 
#define IMB_GHASH_PRE(_mgr, _key, _exp_key)    ((_mgr)->ghash_pre((_key), (_exp_key)))
 
#define IMB_GHASH(_mgr, _exp_key, _src, _len, _tag, _tagl)    ((_mgr)->ghash((_exp_key), (_src), (_len), (_tag), (_tagl)))
 
#define IMB_CHACHA20_POLY1305_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl)
 
#define IMB_CHACHA20_POLY1305_ENC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len)
 
#define IMB_CHACHA20_POLY1305_DEC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len)
 
#define IMB_CHACHA20_POLY1305_ENC_FINALIZE(_mgr, _ctx, _tag, _tagl)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
 
#define IMB_CHACHA20_POLY1305_DEC_FINALIZE(_mgr, _ctx, _tag, _tagl)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
 
#define IMB_ZUC_EEA3_1_BUFFER(_mgr, _key, _iv, _src, _dst, _len)    ((_mgr)->eea3_1_buffer((_key), (_iv), (_src), (_dst), (_len)))
 ZUC EEA3 Confidentiality functions.
 
#define IMB_ZUC_EEA3_4_BUFFER(_mgr, _key, _iv, _src, _dst, _len)    ((_mgr)->eea3_4_buffer((_key), (_iv), (_src), (_dst), (_len)))
 
#define IMB_ZUC_EEA3_N_BUFFER(_mgr, _key, _iv, _src, _dst, _len, _count)    ((_mgr)->eea3_n_buffer((_key), (_iv), (_src), (_dst), (_len), (_count)))
 
#define IMB_ZUC_EIA3_1_BUFFER(_mgr, _key, _iv, _src, _len, _tag)    ((_mgr)->eia3_1_buffer((_key), (_iv), (_src), (_len), (_tag)))
 ZUC EIA3 Integrity function.
 
#define IMB_ZUC_EIA3_N_BUFFER(_mgr, _key, _iv, _src, _len, _tag, _count)    ((_mgr)->eia3_n_buffer((_key), (_iv), (_src), (_len), (_tag), (_count)))
 
#define IMB_KASUMI_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len)    ((_mgr)->f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
 Kasumi byte-level f8 operation on a single buffer.
 
#define IMB_KASUMI_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, _len, _offset)
 Kasumi bit-level f8 operation on a single buffer.
 
#define IMB_KASUMI_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, _src1, _dst1, _len1, _src2, _dst2, _len2)
 Kasumi byte-level f8 operation in parallel on two buffers.
 
#define IMB_KASUMI_F8_3_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _src1, _dst1, _src2, _dst2, _src3, _dst3, _len)
 kasumi byte-level f8 operation in parallel on three buffers
 
#define IMB_KASUMI_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, _src1, _dst1, _src2, _dst2, _src3, _dst3, _src4, _dst4, _len)
 kasumi byte-level f8 operation in parallel on four buffers
 
#define IMB_KASUMI_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count)
 Kasumi f8 operation on N buffers.
 
#define IMB_KASUMI_F9_1_BUFFER(_mgr, _exp_key, _src, _len, _tag)    ((_mgr)->f9_1_buffer((_exp_key), (_src), (_len), (_tag)))
 Kasumi bit-level f9 operation on a single buffer.
 
#define IMB_KASUMI_F9_1_BUFFER_USER(_mgr, _exp_key, _iv, _src, _len, _tag, _dir)
 Kasumi bit-level f9 operation on a single buffer.
 
#define IMB_KASUMI_INIT_F8_KEY_SCHED(_mgr, _key, _exp_key)    ((_mgr)->kasumi_init_f8_key_sched((_key), (_exp_key)))
 
#define IMB_KASUMI_INIT_F9_KEY_SCHED(_mgr, _key, _exp_key)    ((_mgr)->kasumi_init_f9_key_sched((_key), (_exp_key)))
 
#define IMB_KASUMI_KEY_SCHED_SIZE(_mgr)   ((_mgr)->kasumi_key_sched_size())
 
#define IMB_SNOW3G_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, _len, _offset)
 
#define IMB_SNOW3G_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len)    ((_mgr)->snow3g_f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
 
#define IMB_SNOW3G_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, _src1, _dst1, _len1, _src2, _dst2, _len2)
 
#define IMB_SNOW3G_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, _src1, _dst1, _len1, _src2, _dst2, _len2, _src3, _dst3, _len3, _src4, _dst4, _len4)
 
#define IMB_SNOW3G_F8_8_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, _iv5, _iv6, _iv7, _iv8, _src1, _dst1, _len1, _src2, _dst2, _len2, _src3, _dst3, _len3, _src4, _dst4, _len4, _src5, _dst5, _len5, _src6, _dst6, _len6, _src7, _dst7, _len7, _src8, _dst8, _len8)
 
#define IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, _dst, _len)
 
#define IMB_SNOW3G_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count)
 
#define IMB_SNOW3G_F8_N_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, _dst, _len, _count)
 
#define IMB_SNOW3G_F9_1_BUFFER(_mgr, _exp_key, _iv, _src, _len, _tag)    ((_mgr)->snow3g_f9_1_buffer((_exp_key), (_iv), (_src), (_len), (_tag)))
 
#define IMB_SNOW3G_INIT_KEY_SCHED(_mgr, _key, _exp_key)    ((_mgr)->snow3g_init_key_sched((_key), (_exp_key)))
 
#define IMB_SNOW3G_KEY_SCHED_SIZE(_mgr)   ((_mgr)->snow3g_key_sched_size())
 
#define IMB_HEC_32(_mgr, _src)   ((_mgr)->hec_32(_src))
 
#define IMB_HEC_64(_mgr, _src)   ((_mgr)->hec_64(_src))
 
#define IMB_CRC32_ETHERNET_FCS(_mgr, _src, _len)    (_mgr)->crc32_ethernet_fcs(_src, _len)
 
#define IMB_CRC16_X25(_mgr, _src, _len)    (_mgr)->crc16_x25(_src, _len)
 
#define IMB_CRC32_SCTP(_mgr, _src, _len)    (_mgr)->crc32_sctp(_src, _len)
 
#define IMB_CRC24_LTE_A(_mgr, _src, _len)    (_mgr)->crc24_lte_a(_src, _len)
 
#define IMB_CRC24_LTE_B(_mgr, _src, _len)    (_mgr)->crc24_lte_b(_src, _len)
 
#define IMB_CRC16_FP_DATA(_mgr, _src, _len)    (_mgr)->crc16_fp_data(_src, _len)
 
#define IMB_CRC11_FP_HEADER(_mgr, _src, _len)    (_mgr)->crc11_fp_header(_src, _len)
 
#define IMB_CRC7_FP_HEADER(_mgr, _src, _len)    (_mgr)->crc7_fp_header(_src, _len)
 
#define IMB_CRC10_IUUP_DATA(_mgr, _src, _len)    (_mgr)->crc10_iuup_data(_src, _len)
 
#define IMB_CRC6_IUUP_HEADER(_mgr, _src, _len)    (_mgr)->crc6_iuup_header(_src, _len)
 
#define IMB_CRC32_WIMAX_OFDMA_DATA(_mgr, _src, _len)    (_mgr)->crc32_wimax_ofdma_data(_src, _len)
 
#define IMB_CRC8_WIMAX_OFDMA_HCS(_mgr, _src, _len)    (_mgr)->crc8_wimax_ofdma_hcs(_src, _len)
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Typedefs

typedef struct IMB_JOB IMB_JOB
 
typedef struct kasumi_key_sched_s kasumi_key_sched_t
 
typedef void(* init_mb_mgr_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* get_next_job_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* submit_job_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* get_completed_job_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* flush_job_t) (struct IMB_MGR *)
 
typedef uint32_t(* queue_size_t) (struct IMB_MGR *)
 
typedef uint32_t(* burst_fn_t) (struct IMB_MGR *, const uint32_t, struct IMB_JOB **)
 
typedef uint32_t(* submit_cipher_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size)
 
typedef uint32_t(* submit_hash_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_HASH_ALG hash)
 
typedef void(* keyexp_t) (const void *, void *, void *)
 
typedef void(* cmac_subkey_gen_t) (const void *, void *, void *)
 
typedef void(* hash_one_block_t) (const void *, void *)
 
typedef void(* hash_fn_t) (const void *, const uint64_t, void *)
 
typedef void(* xcbc_keyexp_t) (const void *, void *, void *, void *)
 
typedef int(* des_keysched_t) (uint64_t *, const void *)
 
typedef void(* aes_cfb_t) (void *, const void *, const void *, const void *, uint64_t)
 
typedef void(* aes_gcm_enc_dec_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, uint64_t, const uint8_t *, uint8_t const *, uint64_t, uint8_t *, uint64_t)
 
typedef void(* aes_gcm_enc_dec_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, const uint64_t, const uint8_t *, uint8_t const *, const uint64_t, uint8_t *, const uint64_t, const uint64_t)
 
typedef void(* aes_gcm_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, uint8_t const *, uint64_t)
 
typedef void(* aes_gcm_init_var_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t, const uint8_t *, const uint64_t)
 
typedef void(* aes_gcm_enc_dec_update_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint8_t *, uint64_t)
 
typedef void(* aes_gcm_enc_dec_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint64_t)
 
typedef void(* aes_gcm_precomp_t) (struct gcm_key_data *)
 
typedef void(* aes_gcm_pre_t) (const void *, struct gcm_key_data *)
 
typedef void(* aes_gmac_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
 
typedef void(* aes_gmac_update_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
 
typedef void(* aes_gmac_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint64_t)
 
typedef void(* chacha_poly_init_t) (const void *, struct chacha20_poly1305_context_data *, const void *, const void *, const uint64_t)
 
typedef void(* chacha_poly_enc_dec_update_t) (const void *, struct chacha20_poly1305_context_data *, void *, const void *, const uint64_t)
 
typedef void(* chacha_poly_finalize_t) (struct chacha20_poly1305_context_data *, void *, const uint64_t)
 
typedef void(* ghash_t) (const struct gcm_key_data *, const void *, const uint64_t, void *, const uint64_t)
 
typedef void(* zuc_eea3_1_buffer_t) (const void *, const void *, const void *, void *, const uint32_t)
 
typedef void(* zuc_eea3_4_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *)
 
typedef void(* zuc_eea3_n_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *, const uint32_t)
 
typedef void(* zuc_eia3_1_buffer_t) (const void *, const void *, const void *, const uint32_t, uint32_t *)
 
typedef void(* zuc_eia3_n_buffer_t) (const void *const *, const void *const *, const void *const *, const uint32_t *, uint32_t **, const uint32_t)
 
typedef void(* kasumi_f8_1_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_1_buffer_bit_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t, const uint32_t)
 
typedef void(* kasumi_f8_2_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_3_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_4_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_n_buffer_t) (const kasumi_key_sched_t *, const uint64_t *, const void *const *, void **, const uint32_t *, const uint32_t)
 
typedef void(* kasumi_f9_1_buffer_user_t) (const kasumi_key_sched_t *, const uint64_t, const void *, const uint32_t, void *, const uint32_t)
 
typedef void(* kasumi_f9_1_buffer_t) (const kasumi_key_sched_t *, const void *, const uint32_t, void *)
 
typedef int(* kasumi_init_f8_key_sched_t) (const void *, kasumi_key_sched_t *)
 
typedef int(* kasumi_init_f9_key_sched_t) (const void *, kasumi_key_sched_t *)
 
typedef size_t(* kasumi_key_sched_size_t) (void)
 
typedef struct snow3g_key_schedule_s snow3g_key_schedule_t
 
typedef void(* snow3g_f8_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_1_buffer_bit_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t, const uint32_t)
 
typedef void(* snow3g_f8_2_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_4_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_8_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_8_buffer_multikey_t) (const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[])
 
typedef void(* snow3g_f8_n_buffer_t) (const snow3g_key_schedule_t *, const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
 
typedef void(* snow3g_f8_n_buffer_multikey_t) (const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
 
typedef void(* snow3g_f9_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const uint64_t, void *)
 
typedef int(* snow3g_init_key_sched_t) (const void *, snow3g_key_schedule_t *)
 
typedef size_t(* snow3g_key_sched_size_t) (void)
 
typedef uint32_t(* hec_32_t) (const uint8_t *)
 
typedef uint64_t(* hec_64_t) (const uint8_t *)
 
typedef uint32_t(* crc32_fn_t) (const void *, const uint64_t)
 
typedef void(* aes_ecb_quic_t) (const void *, const void *, void *out, uint64_t)
 
typedef struct IMB_MGR IMB_MGR
 
+ + + + + + + + + + + + + + + + + + + +

+Enumerations

enum  IMB_ARCH {
+  IMB_ARCH_NONE = 0 +, IMB_ARCH_NOAESNI +, IMB_ARCH_SSE +, IMB_ARCH_AVX +,
+  IMB_ARCH_AVX2 +, IMB_ARCH_AVX512 +, IMB_ARCH_NUM +
+ }
 
enum  IMB_STATUS {
+  IMB_STATUS_BEING_PROCESSED = 0 +, IMB_STATUS_COMPLETED_CIPHER = 1 +, IMB_STATUS_COMPLETED_AUTH = 2 +, IMB_STATUS_COMPLETED = 3 +,
+  IMB_STATUS_INVALID_ARGS = 4 +, IMB_STATUS_INTERNAL_ERROR +, IMB_STATUS_ERROR +
+ }
 
enum  IMB_ERR {
+  IMB_ERR_MIN = 2000 +, IMB_ERR_NULL_MBMGR +, IMB_ERR_JOB_NULL_SRC +, IMB_ERR_JOB_NULL_DST +,
+  IMB_ERR_JOB_NULL_KEY +, IMB_ERR_JOB_NULL_IV +, IMB_ERR_JOB_NULL_AUTH +, IMB_ERR_JOB_NULL_AAD +,
+  IMB_ERR_JOB_CIPH_LEN +, IMB_ERR_JOB_AUTH_LEN +, IMB_ERR_JOB_IV_LEN +, IMB_ERR_JOB_KEY_LEN +,
+  IMB_ERR_JOB_AUTH_TAG_LEN +, IMB_ERR_JOB_AAD_LEN +, IMB_ERR_JOB_SRC_OFFSET +, IMB_ERR_JOB_CHAIN_ORDER +,
+  IMB_ERR_CIPH_MODE +, IMB_ERR_HASH_ALGO +, IMB_ERR_JOB_NULL_AUTH_KEY +, IMB_ERR_JOB_NULL_SGL_CTX +,
+  IMB_ERR_JOB_NULL_NEXT_IV +, IMB_ERR_JOB_PON_PLI +, IMB_ERR_NULL_SRC +, IMB_ERR_NULL_DST +,
+  IMB_ERR_NULL_KEY +, IMB_ERR_NULL_EXP_KEY +, IMB_ERR_NULL_IV +, IMB_ERR_NULL_AUTH +,
+  IMB_ERR_NULL_AAD +, IMB_ERR_CIPH_LEN +, IMB_ERR_AUTH_LEN +, IMB_ERR_IV_LEN +,
+  IMB_ERR_KEY_LEN +, IMB_ERR_AUTH_TAG_LEN +, IMB_ERR_AAD_LEN +, IMB_ERR_SRC_OFFSET +,
+  IMB_ERR_NULL_AUTH_KEY +, IMB_ERR_NULL_CTX +, IMB_ERR_NO_AESNI_EMU +, IMB_ERR_JOB_NULL_HMAC_OPAD +,
+  IMB_ERR_JOB_NULL_HMAC_IPAD +, IMB_ERR_JOB_NULL_XCBC_K1_EXP +, IMB_ERR_JOB_NULL_XCBC_K2 +, IMB_ERR_JOB_NULL_XCBC_K3 +,
+  IMB_ERR_JOB_CIPH_DIR +, IMB_ERR_JOB_NULL_GHASH_INIT_TAG +, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR +, IMB_ERR_NULL_JOB +,
+  IMB_ERR_QUEUE_SPACE +, IMB_ERR_NULL_BURST +, IMB_ERR_BURST_SIZE +, IMB_ERR_BURST_OOO +,
+  IMB_ERR_SELFTEST +, IMB_ERR_BURST_SUITE_ID +, IMB_ERR_MAX +
+ }
 
enum  IMB_CIPHER_MODE {
+  IMB_CIPHER_CBC = 1 +, IMB_CIPHER_CNTR +, IMB_CIPHER_NULL +, IMB_CIPHER_DOCSIS_SEC_BPI +,
+  IMB_CIPHER_GCM +, IMB_CIPHER_CUSTOM +, IMB_CIPHER_DES +, IMB_CIPHER_DOCSIS_DES +,
+  IMB_CIPHER_CCM +, IMB_CIPHER_DES3 +, IMB_CIPHER_PON_AES_CNTR +, IMB_CIPHER_ECB +,
+  IMB_CIPHER_CNTR_BITLEN +, IMB_CIPHER_ZUC_EEA3 +, IMB_CIPHER_SNOW3G_UEA2_BITLEN +, IMB_CIPHER_KASUMI_UEA1_BITLEN +,
+  IMB_CIPHER_CBCS_1_9 +, IMB_CIPHER_CHACHA20 +, IMB_CIPHER_CHACHA20_POLY1305 +, IMB_CIPHER_CHACHA20_POLY1305_SGL +,
+  IMB_CIPHER_SNOW_V +, IMB_CIPHER_SNOW_V_AEAD +, IMB_CIPHER_GCM_SGL +, IMB_CIPHER_NUM +
+ }
 
enum  IMB_CIPHER_DIRECTION { IMB_DIR_ENCRYPT = 1 +, IMB_DIR_DECRYPT + }
 
enum  IMB_HASH_ALG {
+  IMB_AUTH_HMAC_SHA_1 = 1 +, IMB_AUTH_HMAC_SHA_224 +, IMB_AUTH_HMAC_SHA_256 +, IMB_AUTH_HMAC_SHA_384 +,
+  IMB_AUTH_HMAC_SHA_512 +, IMB_AUTH_AES_XCBC +, IMB_AUTH_MD5 +, IMB_AUTH_NULL +,
+  IMB_AUTH_AES_GMAC +, IMB_AUTH_CUSTOM +, IMB_AUTH_AES_CCM +, IMB_AUTH_AES_CMAC +,
+  IMB_AUTH_SHA_1 +, IMB_AUTH_SHA_224 +, IMB_AUTH_SHA_256 +, IMB_AUTH_SHA_384 +,
+  IMB_AUTH_SHA_512 +, IMB_AUTH_AES_CMAC_BITLEN +, IMB_AUTH_PON_CRC_BIP +, IMB_AUTH_ZUC_EIA3_BITLEN +,
+  IMB_AUTH_DOCSIS_CRC32 +, IMB_AUTH_SNOW3G_UIA2_BITLEN +, IMB_AUTH_KASUMI_UIA1 +, IMB_AUTH_AES_GMAC_128 +,
+  IMB_AUTH_AES_GMAC_192 +, IMB_AUTH_AES_GMAC_256 +, IMB_AUTH_AES_CMAC_256 +, IMB_AUTH_POLY1305 +,
+  IMB_AUTH_CHACHA20_POLY1305 +, IMB_AUTH_CHACHA20_POLY1305_SGL +, IMB_AUTH_ZUC256_EIA3_BITLEN +, IMB_AUTH_SNOW_V_AEAD +,
+  IMB_AUTH_GCM_SGL +, IMB_AUTH_CRC32_ETHERNET_FCS +, IMB_AUTH_CRC32_SCTP +, IMB_AUTH_CRC32_WIMAX_OFDMA_DATA +,
+  IMB_AUTH_CRC24_LTE_A +, IMB_AUTH_CRC24_LTE_B +, IMB_AUTH_CRC16_X25 +, IMB_AUTH_CRC16_FP_DATA +,
+  IMB_AUTH_CRC11_FP_HEADER +, IMB_AUTH_CRC10_IUUP_DATA +, IMB_AUTH_CRC8_WIMAX_OFDMA_HCS +, IMB_AUTH_CRC7_FP_HEADER +,
+  IMB_AUTH_CRC6_IUUP_HEADER +, IMB_AUTH_GHASH +, IMB_AUTH_NUM +
+ }
 
enum  IMB_CHAIN_ORDER { IMB_ORDER_CIPHER_HASH = 1 +, IMB_ORDER_HASH_CIPHER + }
 
enum  IMB_KEY_SIZE_BYTES { IMB_KEY_64_BYTES = 8 +, IMB_KEY_128_BYTES = 16 +, IMB_KEY_192_BYTES = 24 +, IMB_KEY_256_BYTES = 32 + }
 
enum  IMB_SGL_STATE { IMB_SGL_INIT = 0 +, IMB_SGL_UPDATE +, IMB_SGL_COMPLETE +, IMB_SGL_ALL + }
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Functions

IMB_DLL_EXPORT const char * imb_get_version_str (void)
 Get library version in string format.
 
IMB_DLL_EXPORT unsigned imb_get_version (void)
 Get library version in numerical format.
 
IMB_DLL_EXPORT int imb_get_errno (IMB_MGR *mb_mgr)
 API to get error status.
 
IMB_DLL_EXPORT const char * imb_get_strerror (int errnum)
 API to get description for errnum.
 
IMB_DLL_EXPORT IMB_MGR * alloc_mb_mgr (uint64_t flags)
 Allocates memory for multi-buffer manager instance.
 
IMB_DLL_EXPORT void free_mb_mgr (IMB_MGR *ptr)
 Frees memory allocated previously by alloc_mb_mgr()
 
IMB_DLL_EXPORT size_t imb_get_mb_mgr_size (void)
 Calculates necessary memory size for IMB_MGR.
 
IMB_DLL_EXPORT IMB_MGR * imb_set_pointers_mb_mgr (void *ptr, const uint64_t flags, const unsigned reset_mgr)
 Initializes IMB_MGR pointers to out-of-order managers with use of externally allocated memory.
 
IMB_DLL_EXPORT uint64_t imb_get_feature_flags (void)
 Retrieves the bitmask with the features supported by the library, without having to allocate/initialize IMB_MGR;.
 
IMB_DLL_EXPORT void init_mb_mgr_avx (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure.
 
IMB_DLL_EXPORT void init_mb_mgr_avx2 (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure.
 
IMB_DLL_EXPORT void init_mb_mgr_avx512 (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure.
 
IMB_DLL_EXPORT void init_mb_mgr_sse (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_avx (IMB_MGR *state)
 Submit job for processing after validating.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_avx2 (IMB_MGR *state)
 Submit job for processing after validating.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_avx512 (IMB_MGR *state)
 Submit job for processing after validating.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_sse (IMB_MGR *state)
 Submit job for processing after validating.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx (IMB_MGR *state)
 Submit job for processing without validating.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx2 (IMB_MGR *state)
 Submit job for processing without validating.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx512 (IMB_MGR *state)
 Submit job for processing without validating.
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_sse (IMB_MGR *state)
 Submit job for processing without validating.
 
IMB_DLL_EXPORT IMB_JOB * flush_job_avx (IMB_MGR *state)
 Force processing until next job in queue is completed.
 
IMB_DLL_EXPORT IMB_JOB * flush_job_avx2 (IMB_MGR *state)
 Force processing until next job in queue is completed.
 
IMB_DLL_EXPORT IMB_JOB * flush_job_avx512 (IMB_MGR *state)
 Force processing until next job in queue is completed.
 
IMB_DLL_EXPORT IMB_JOB * flush_job_sse (IMB_MGR *state)
 Force processing until next job in queue is completed.
 
IMB_DLL_EXPORT uint32_t queue_size_avx (IMB_MGR *state)
 Get number of jobs queued to be processed.
 
IMB_DLL_EXPORT uint32_t queue_size_avx2 (IMB_MGR *state)
 Get number of jobs queued to be processed.
 
IMB_DLL_EXPORT uint32_t queue_size_avx512 (IMB_MGR *state)
 Get number of jobs queued to be processed.
 
IMB_DLL_EXPORT uint32_t queue_size_sse (IMB_MGR *state)
 Get number of jobs queued to be processed.
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx (IMB_MGR *state)
 Get next completed job.
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx2 (IMB_MGR *state)
 Get next completed job.
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx512 (IMB_MGR *state)
 Get next completed job.
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_sse (IMB_MGR *state)
 Get next completed job.
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx (IMB_MGR *state)
 Get next available job.
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx2 (IMB_MGR *state)
 Get next available job.
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx512 (IMB_MGR *state)
 Get next available job.
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_sse (IMB_MGR *state)
 Get next available job.
 
IMB_DLL_EXPORT void init_mb_mgr_auto (IMB_MGR *state, IMB_ARCH *arch)
 Automatically initialize most performant Multi-buffer manager based on CPU features.
 
IMB_DLL_EXPORT int des_key_schedule (uint64_t *ks, const void *key)
 DES key schedule set up.
 
IMB_DLL_EXPORT void imb_hmac_ipad_opad (struct IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pkey, const size_t key_len, void *ipad_hash, void *opad_hash)
 Ipad Opad padding for HMAC.
 
IMB_DLL_EXPORT void des_cfb_one (void *out, const void *in, const uint64_t *iv, const uint64_t *ks, const int len)
 DES-CFB Encrypt/Decrypt up to one block.
 
IMB_DLL_EXPORT void md5_one_block_sse (const void *data, void *digest)
 
IMB_DLL_EXPORT void md5_one_block_avx (const void *data, void *digest)
 
IMB_DLL_EXPORT void md5_one_block_avx2 (const void *data, void *digest)
 
IMB_DLL_EXPORT void md5_one_block_avx512 (const void *data, void *digest)
 
IMB_DLL_EXPORT void aes_keyexp_128_sse (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_avx (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_avx2 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_avx512 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_sse (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_avx (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_avx2 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_avx512 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_sse (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_avx (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_avx2 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_avx512 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_sse (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_sse (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_sse (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_sse (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2 (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512 (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2 (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512 (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_gcm_enc_128_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Decryption.
 
IMB_DLL_EXPORT void aes_gcm_init_128_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_192_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_256_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-128 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-192 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-256 encryption message.
 
IMB_DLL_EXPORT void aes_gcm_precomp_128_sse (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-128 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-128 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-128 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_192_sse (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-192 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-192 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-192 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_256_sse (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-256 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-256 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-256 HashKey constants.
 
IMB_DLL_EXPORT void aes_gcm_pre_128_sse (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-128 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-128 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-128 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_192_sse (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-192 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-192 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-192 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_256_sse (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-256 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-256 key data.
 
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-256 key data.
 
IMB_DLL_EXPORT int zuc_eea3_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of ZUC-EEA3 Initialization Vector.
 
IMB_DLL_EXPORT int zuc_eia3_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of ZUC-EIA3 Initialization Vector.
 
IMB_DLL_EXPORT int kasumi_f8_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of KASUMI F8 Initialization Vector.
 
IMB_DLL_EXPORT int kasumi_f9_iv_gen (const uint32_t count, const uint32_t fresh, void *iv_ptr)
 Generation of KASUMI F9 Initialization Vector.
 
IMB_DLL_EXPORT int snow3g_f8_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of SNOW3G F8 Initialization Vector.
 
IMB_DLL_EXPORT int snow3g_f9_iv_gen (const uint32_t count, const uint32_t fresh, const uint8_t dir, void *iv_ptr)
 Generation of SNOW3G F9 Initialization Vector.
 
IMB_DLL_EXPORT void imb_clear_mem (void *mem, const size_t size)
 Force clearing/zeroing of memory.
 
IMB_DLL_EXPORT void imb_quic_aes_gcm (IMB_MGR *state, const struct gcm_key_data *key_data, const IMB_KEY_SIZE_BYTES key_size, const IMB_CIPHER_DIRECTION cipher_dir, void *dst_ptr_array[], const void *const src_ptr_array[], const uint64_t len_array[], const void *const iv_ptr_array[], const void *const aad_ptr_array[], const uint64_t aad_len, void *tag_ptr_array[], const uint64_t tag_len, const uint64_t num_packets)
 Batch of GCM encrypt/decrypt operations with the same key.
 
IMB_DLL_EXPORT void imb_quic_hp_aes_ecb (IMB_MGR *state, const void *exp_key_data, void *dst_ptr_array[], const void *const src_ptr_array[], const uint64_t num_packets, const IMB_KEY_SIZE_BYTES key_size)
 Batch of AES-ECB encrypt/decrypt operations with the same key.
 
IMB_DLL_EXPORT uint32_t imb_set_session (IMB_MGR *state, IMB_JOB *job)
 Sets up suite_id and session_id fields for selected cipher suite in provided job structure.
 
+

Macro Definition Documentation

+ +

◆ __forceinline

+ +
+
+ + + + +
#define __forceinline    static __forceinline
+
+ +
+
+ +

◆ DECLARE_ALIGNED

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define DECLARE_ALIGNED( decl,
 alignval 
)    __declspec(align(alignval)) decl
+
+

Macros for aligning data structures and function inlines

+ +
+
+ +

◆ IMB_AES128_CFB_ONE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_CFB_ONE( _mgr,
 _dst,
 _src,
 _iv,
 _exp_key,
 _len 
)    ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
+
+ +

AES-CFB-128 Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[out]_dstPlaintext/Ciphertext output
[in]_srcPlaintext/Ciphertext input
[in]_ivPointer to 16 byte IV
[in]_exp_keyPointer to expanded AES keys
[in]_lenLength of data in bytes
+
+
+ +
+
+ +

◆ IMB_AES128_GCM_DEC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_DEC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm128_dec((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES128_GCM_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_DEC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm128_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_DEC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm128_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_ENC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_ENC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm128_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES128_GCM_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_ENC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm128_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_ENC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm128_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)    ((_mgr)->gcm128_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_INIT_VAR_IV

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_INIT_VAR_IV( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->gcm128_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
(_aad), (_aadl)))
+
+
+
+ +

◆ IMB_AES128_GCM_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->gcm128_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_PRECOMP

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_PRECOMP( _mgr,
 _key 
)    ((_mgr)->gcm128_precomp((_key)))
+
+ +
+
+ +

◆ IMB_AES128_GMAC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GMAC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gmac128_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES128_GMAC_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GMAC_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl 
)    ((_mgr)->gmac128_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
+ +
+
+ +

◆ IMB_AES128_GMAC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GMAC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _src,
 _len 
)    ((_mgr)->gmac128_update((_exp_key), (_ctx), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_DEC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_DEC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm192_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
(_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES192_GCM_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_DEC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm192_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_DEC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm192_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_ENC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_ENC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm192_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES192_GCM_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_ENC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm192_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_ENC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm192_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)    ((_mgr)->gcm192_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_INIT_VAR_IV

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_INIT_VAR_IV( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->gcm192_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
(_aad), (_aadl)))
+
+
+
+ +

◆ IMB_AES192_GCM_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->gcm192_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_PRECOMP

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_PRECOMP( _mgr,
 _key 
)    ((_mgr)->gcm192_precomp((_key)))
+
+ +
+
+ +

◆ IMB_AES192_GMAC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GMAC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gmac192_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES192_GMAC_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GMAC_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl 
)    ((_mgr)->gmac192_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
+ +
+
+ +

◆ IMB_AES192_GMAC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GMAC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _src,
 _len 
)    ((_mgr)->gmac192_update((_exp_key), (_ctx), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES256_CFB_ONE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_CFB_ONE( _mgr,
 _dst,
 _src,
 _iv,
 _exp_key,
 _len 
)    ((_mgr)->aes256_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
+
+ +

AES-CFB-256 Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[out]_dstPlaintext/Ciphertext output
[in]_srcPlaintext/Ciphertext input
[in]_ivPointer to 16 byte IV
[in]_exp_keyPointer to expanded AES keys
[in]_lenLength of data in bytes
+
+
+ +
+
+ +

◆ IMB_AES256_GCM_DEC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_DEC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm256_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
(_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES256_GCM_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_DEC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm256_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_DEC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm256_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_ENC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_ENC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm256_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES256_GCM_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_ENC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm256_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_ENC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm256_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)    ((_mgr)->gcm256_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_INIT_VAR_IV

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_INIT_VAR_IV( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->gcm256_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
(_aad), (_aadl)))
+
+
+
+ +

◆ IMB_AES256_GCM_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->gcm256_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_PRECOMP

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_PRECOMP( _mgr,
 _key 
)    ((_mgr)->gcm256_precomp((_key)))
+
+ +
+
+ +

◆ IMB_AES256_GMAC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GMAC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gmac256_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES256_GMAC_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GMAC_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl 
)    ((_mgr)->gmac256_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
+ +
+
+ +

◆ IMB_AES256_GMAC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GMAC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _src,
 _len 
)    ((_mgr)->gmac256_update((_exp_key), (_ctx), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_AES_BLOCK_SIZE   16
+
+ +
+
+ +

◆ IMB_AES_CMAC_SUBKEY_GEN_128

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_CMAC_SUBKEY_GEN_128( _mgr,
 _exp_key,
 _key1,
 _key2 
)    ((_mgr)->cmac_subkey_gen_128((_exp_key), (_key1), (_key2)))
+
+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyInput expanded AES-128-CMAC key
[out]_key1Subkey 1
[out]_key2Subkey 2
+
+
+ +
+
+ +

◆ IMB_AES_CMAC_SUBKEY_GEN_256

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_CMAC_SUBKEY_GEN_256( _mgr,
 _exp_key,
 _key1,
 _key2 
)    ((_mgr)->cmac_subkey_gen_256((_exp_key), (_key1), (_key2)))
+
+

Generate AES-256-CMAC subkeys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyInput expanded AES-256-CMAC key
[out]_key1Subkey 1
[out]_key2Subkey 2
+
+
+ +
+
+ +

◆ IMB_AES_KEYEXP_128

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_KEYEXP_128( _mgr,
 _key,
 _enc_exp_key,
 _dec_exp_key 
)    ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key)))
+
+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-128 key
[out]_enc_exp_keyAES-128 encryption expansion key
[out]_dec_exp_keyAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ IMB_AES_KEYEXP_192

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_KEYEXP_192( _mgr,
 _key,
 _enc_exp_key,
 _dec_exp_key 
)    ((_mgr)->keyexp_192((_key), (_enc_exp_key), (_dec_exp_key)))
+
+

Generate encryption/decryption AES-192 expansion keys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-192 key
[out]_enc_exp_keyAES-192 encryption expansion key
[out]_dec_exp_keyAES-192 decryption expansion key
+
+
+ +
+
+ +

◆ IMB_AES_KEYEXP_256

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_KEYEXP_256( _mgr,
 _key,
 _enc_exp_key,
 _dec_exp_key 
)    ((_mgr)->keyexp_256((_key), (_enc_exp_key), (_dec_exp_key)))
+
+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-256 key
[out]_enc_exp_keyAES-256 encryption expansion key
[out]_dec_exp_keyAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ IMB_AES_XCBC_KEYEXP

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_XCBC_KEYEXP( _mgr,
 _key,
 _exp_key,
 _exp_key2,
 _exp_key3 
)    ((_mgr)->xcbc_keyexp((_key), (_exp_key), (_exp_key2), (_exp_key3)))
+
+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-128-XCBC key
[out]_exp_keyk1 expansion key
[out]_exp_key2k2 expansion key
[out]_exp_key3k3 expansion key
+
+
+ +
+
+ +

◆ IMB_ASSERT

+ +
+
+ + + + + + + + +
#define IMB_ASSERT( x)
+
+

Custom ASSERT and DIM macros

+ +
+
+ +

◆ IMB_CHACHA20_POLY1305_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_DEC_FINALIZE( _mgr,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_CHACHA20_POLY1305_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_DEC_UPDATE( _mgr,
 _key,
 _ctx,
 _dst,
 _src,
 _len 
)
+
+Value:
((_mgr)->chacha20_poly1305_dec_update((_key), (_ctx), (_dst), (_src), \
+
(_len)))
+
+
+
+ +

◆ IMB_CHACHA20_POLY1305_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_ENC_FINALIZE( _mgr,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_CHACHA20_POLY1305_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_ENC_UPDATE( _mgr,
 _key,
 _ctx,
 _dst,
 _src,
 _len 
)
+
+Value:
((_mgr)->chacha20_poly1305_enc_update((_key), (_ctx), (_dst), (_src), \
+
(_len)))
+
+
+
+ +

◆ IMB_CHACHA20_POLY1305_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_INIT( _mgr,
 _key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->chacha20_poly1305_init((_key), (_ctx), (_iv), (_aad), \
+
(_aadl)))
+
+
+
+ +

◆ IMB_CPUFLAGS_AVX

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX   (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX)
+
+ +
+
+ +

◆ IMB_CPUFLAGS_AVX2

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX2
+
+Value:
(IMB_CPUFLAGS_AVX | IMB_FEATURE_AVX2 | \
+
IMB_FEATURE_BMI2)
+
IMB_CPUFLAGS_AVX
#define IMB_CPUFLAGS_AVX
Definition intel-ipsec-mb.h:961
+
IMB_FEATURE_AVX2
#define IMB_FEATURE_AVX2
Definition intel-ipsec-mb.h:932
+
IMB_FEATURE_BMI2
#define IMB_FEATURE_BMI2
Definition intel-ipsec-mb.h:947
+
+
+
+ +

◆ IMB_CPUFLAGS_AVX2_T2

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX2_T2
+
+Value:
(IMB_CPUFLAGS_AVX2 | IMB_FEATURE_SHANI | \
+
IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ | \
+
IMB_FEATURE_GFNI)
+
IMB_FEATURE_VPCLMULQDQ
#define IMB_FEATURE_VPCLMULQDQ
Definition intel-ipsec-mb.h:942
+
IMB_FEATURE_GFNI
#define IMB_FEATURE_GFNI
Definition intel-ipsec-mb.h:945
+
IMB_CPUFLAGS_AVX2
#define IMB_CPUFLAGS_AVX2
Definition intel-ipsec-mb.h:962
+
IMB_FEATURE_SHANI
#define IMB_FEATURE_SHANI
Definition intel-ipsec-mb.h:926
+
IMB_FEATURE_VAES
#define IMB_FEATURE_VAES
Definition intel-ipsec-mb.h:941
+
+
+
+ +

◆ IMB_CPUFLAGS_AVX2_T3

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX2_T3   (IMB_CPUFLAGS_AVX2_T2 | IMB_FEATURE_AVX_IFMA)
+
+ +
+
+ +

◆ IMB_CPUFLAGS_AVX512

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX512   (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX)
+
+ +
+
+ +

◆ IMB_CPUFLAGS_AVX512_T2

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX512_T2
+
+Value:
(IMB_CPUFLAGS_AVX512 | IMB_FEATURE_VAES | \
+
IMB_FEATURE_VPCLMULQDQ | IMB_FEATURE_GFNI | \
+
IMB_FEATURE_AVX512_IFMA | IMB_FEATURE_SHANI)
+
IMB_FEATURE_AVX512_IFMA
#define IMB_FEATURE_AVX512_IFMA
Definition intel-ipsec-mb.h:946
+
IMB_CPUFLAGS_AVX512
#define IMB_CPUFLAGS_AVX512
Definition intel-ipsec-mb.h:964
+
+
+
+ +

◆ IMB_CPUFLAGS_AVX_T2

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX_T2
+
+Value:
(IMB_CPUFLAGS_AVX | IMB_FEATURE_SHANI | \
+
IMB_FEATURE_GFNI)
+
+
+
+ +

◆ IMB_CPUFLAGS_NO_AESNI

+ +
+
+ + + + +
#define IMB_CPUFLAGS_NO_AESNI   (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV)
+
+

CPU flags needed for each implementation

+ +
+
+ +

◆ IMB_CPUFLAGS_SSE

+ +
+
+ + + + +
#define IMB_CPUFLAGS_SSE
+
+Value:
(IMB_CPUFLAGS_NO_AESNI | IMB_FEATURE_AESNI | \
+
IMB_FEATURE_PCLMULQDQ)
+
IMB_FEATURE_AESNI
#define IMB_FEATURE_AESNI
Definition intel-ipsec-mb.h:927
+
IMB_FEATURE_PCLMULQDQ
#define IMB_FEATURE_PCLMULQDQ
Definition intel-ipsec-mb.h:928
+
IMB_CPUFLAGS_NO_AESNI
#define IMB_CPUFLAGS_NO_AESNI
Definition intel-ipsec-mb.h:956
+
+
+
+ +

◆ IMB_CPUFLAGS_SSE_T2

+ +
+
+ + + + +
#define IMB_CPUFLAGS_SSE_T2   (IMB_CPUFLAGS_SSE | IMB_FEATURE_SHANI)
+
+ +
+
+ +

◆ IMB_CPUFLAGS_SSE_T3

+ +
+
+ + + + +
#define IMB_CPUFLAGS_SSE_T3   (IMB_CPUFLAGS_SSE_T2 | IMB_FEATURE_GFNI)
+
+ +
+
+ +

◆ IMB_CRC10_IUUP_DATA

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC10_IUUP_DATA( _mgr,
 _src,
 _len 
)    (_mgr)->crc10_iuup_data(_src, _len)
+
+

IUUP CRC10 function (3GPP TS 25.415)

+ +
+
+ +

◆ IMB_CRC11_FP_HEADER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC11_FP_HEADER( _mgr,
 _src,
 _len 
)    (_mgr)->crc11_fp_header(_src, _len)
+
+

Framing Protocol CRC11 function (3GPP TS 25.435, 3GPP TS 25.427)

+ +
+
+ +

◆ IMB_CRC16_FP_DATA

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC16_FP_DATA( _mgr,
 _src,
 _len 
)    (_mgr)->crc16_fp_data(_src, _len)
+
+

Framing Protocol CRC16 function (3GPP TS 25.435, 3GPP TS 25.427)

+ +
+
+ +

◆ IMB_CRC16_X25

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC16_X25( _mgr,
 _src,
 _len 
)    (_mgr)->crc16_x25(_src, _len)
+
+

CRC16 X25 function

+ +
+
+ +

◆ IMB_CRC24_LTE_A

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC24_LTE_A( _mgr,
 _src,
 _len 
)    (_mgr)->crc24_lte_a(_src, _len)
+
+

LTE CRC24A function

+ +
+
+ +

◆ IMB_CRC24_LTE_B

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC24_LTE_B( _mgr,
 _src,
 _len 
)    (_mgr)->crc24_lte_b(_src, _len)
+
+

LTE CRC24B function

+ +
+
+ +

◆ IMB_CRC32_ETHERNET_FCS

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC32_ETHERNET_FCS( _mgr,
 _src,
 _len 
)    (_mgr)->crc32_ethernet_fcs(_src, _len)
+
+

CRC32 Ethernet FCS function

+ +
+
+ +

◆ IMB_CRC32_SCTP

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC32_SCTP( _mgr,
 _src,
 _len 
)    (_mgr)->crc32_sctp(_src, _len)
+
+

CRC32 SCTP function

+ +
+
+ +

◆ IMB_CRC32_WIMAX_OFDMA_DATA

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC32_WIMAX_OFDMA_DATA( _mgr,
 _src,
 _len 
)    (_mgr)->crc32_wimax_ofdma_data(_src, _len)
+
+

WIMAX OFDMA DATA CRC32 function (IEEE 802.16)

+ +
+
+ +

◆ IMB_CRC6_IUUP_HEADER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC6_IUUP_HEADER( _mgr,
 _src,
 _len 
)    (_mgr)->crc6_iuup_header(_src, _len)
+
+

IUUP CRC6 function (3GPP TS 25.415)

+ +
+
+ +

◆ IMB_CRC7_FP_HEADER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC7_FP_HEADER( _mgr,
 _src,
 _len 
)    (_mgr)->crc7_fp_header(_src, _len)
+
+

Framing Protocol CRC7 function (3GPP TS 25.435, 3GPP TS 25.427)

+ +
+
+ +

◆ IMB_CRC8_WIMAX_OFDMA_HCS

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC8_WIMAX_OFDMA_HCS( _mgr,
 _src,
 _len 
)    (_mgr)->crc8_wimax_ofdma_hcs(_src, _len)
+
+

WIMAX OFDMA HCS CRC8 function (IEEE 802.16)

+ +
+
+ +

◆ IMB_DES_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_DES_BLOCK_SIZE   8
+
+ +
+
+ +

◆ IMB_DES_KEY_SCHED_SIZE

+ +
+
+ + + + +
#define IMB_DES_KEY_SCHED_SIZE   (16 * 8)
+
+

Algorithm constants 16 rounds x 8 bytes

+ +
+
+ +

◆ IMB_DES_KEYSCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_DES_KEYSCHED( _mgr,
 _exp_key,
 _key 
)    ((_mgr)->des_key_sched((_exp_key), (_key)))
+
+ +
+
+ +

◆ IMB_DIM

+ +
+
+ + + + + + + + +
#define IMB_DIM( x)   (sizeof(x) / sizeof(x[0]))
+
+ +
+
+ +

◆ IMB_DLL_EXPORT

+ +
+
+ + + + +
#define IMB_DLL_EXPORT
+
+

Windows DLL export is done via DEF file

+ +
+
+ +

◆ IMB_DLL_LOCAL

+ +
+
+ + + + +
#define IMB_DLL_LOCAL
+
+ +
+
+ +

◆ IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE

+ +
+
+ + + + +
#define IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE   14
+
+

Minimum Ethernet frame size to calculate CRC32 Source Address (6 bytes) + Destination Address (6 bytes) + Type/Len (2 bytes)

+ +
+
+ +

◆ IMB_DOCSIS_CRC32_TAG_SIZE

+ +
+
+ + + + +
#define IMB_DOCSIS_CRC32_TAG_SIZE   4
+
+ +
+
+ +

◆ IMB_FEATURE_AESNI

+ +
+
+ + + + +
#define IMB_FEATURE_AESNI   (1ULL << 1)
+
+ +
+
+ +

◆ IMB_FEATURE_AESNI_EMU

+ +
+
+ + + + +
#define IMB_FEATURE_AESNI_EMU   (1ULL << 19)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX

+ +
+
+ + + + +
#define IMB_FEATURE_AVX   (1ULL << 5)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX2

+ +
+
+ + + + +
#define IMB_FEATURE_AVX2   (1ULL << 6)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512_IFMA

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512_IFMA   (1ULL << 17)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512_SKX

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512_SKX
+
+Value:
(IMB_FEATURE_AVX512F | IMB_FEATURE_AVX512DQ | \
+
IMB_FEATURE_AVX512CD | IMB_FEATURE_AVX512BW | \
+
IMB_FEATURE_AVX512VL)
+
IMB_FEATURE_AVX512BW
#define IMB_FEATURE_AVX512BW
Definition intel-ipsec-mb.h:936
+
IMB_FEATURE_AVX512VL
#define IMB_FEATURE_AVX512VL
Definition intel-ipsec-mb.h:937
+
IMB_FEATURE_AVX512DQ
#define IMB_FEATURE_AVX512DQ
Definition intel-ipsec-mb.h:934
+
IMB_FEATURE_AVX512F
#define IMB_FEATURE_AVX512F
Definition intel-ipsec-mb.h:933
+
IMB_FEATURE_AVX512CD
#define IMB_FEATURE_AVX512CD
Definition intel-ipsec-mb.h:935
+
+
+
+ +

◆ IMB_FEATURE_AVX512BW

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512BW   (1ULL << 10)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512CD

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512CD   (1ULL << 9)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512DQ

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512DQ   (1ULL << 8)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512F

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512F   (1ULL << 7)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512VL

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512VL   (1ULL << 11)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX_IFMA

+ +
+
+ + + + +
#define IMB_FEATURE_AVX_IFMA   (1ULL << 22)
+
+ +
+
+ +

◆ IMB_FEATURE_BMI2

+ +
+
+ + + + +
#define IMB_FEATURE_BMI2   (1ULL << 18)
+
+ +
+
+ +

◆ IMB_FEATURE_CMOV

+ +
+
+ + + + +
#define IMB_FEATURE_CMOV   (1ULL << 3)
+
+ +
+
+ +

◆ IMB_FEATURE_GFNI

+ +
+
+ + + + +
#define IMB_FEATURE_GFNI   (1ULL << 16)
+
+ +
+
+ +

◆ IMB_FEATURE_PCLMULQDQ

+ +
+
+ + + + +
#define IMB_FEATURE_PCLMULQDQ   (1ULL << 2)
+
+ +
+
+ +

◆ IMB_FEATURE_SAFE_DATA

+ +
+
+ + + + +
#define IMB_FEATURE_SAFE_DATA   (1ULL << 14)
+
+ +
+
+ +

◆ IMB_FEATURE_SAFE_PARAM

+ +
+
+ + + + +
#define IMB_FEATURE_SAFE_PARAM   (1ULL << 15)
+
+ +
+
+ +

◆ IMB_FEATURE_SELF_TEST

+ +
+
+ + + + +
#define IMB_FEATURE_SELF_TEST   (1ULL << 20) /* self-test feature present */
+
+ +
+
+ +

◆ IMB_FEATURE_SELF_TEST_PASS

+ +
+
+ + + + +
#define IMB_FEATURE_SELF_TEST_PASS   (1ULL << 21) /* self-test passed */
+
+ +
+
+ +

◆ IMB_FEATURE_SHANI

+ +
+
+ + + + +
#define IMB_FEATURE_SHANI   (1ULL << 0)
+
+

Multi-buffer manager detected features

    +
  • if bit is set then hardware supports given extension
    • +
  • valid after call to init_mb_mgr() or alloc_mb_mgr()
    • +
  • some HW supported features can be disabled via IMB_FLAG_xxx (see above)
    • +
+ +
+
+ +

◆ IMB_FEATURE_SSE4_2

+ +
+
+ + + + +
#define IMB_FEATURE_SSE4_2   (1ULL << 4)
+
+ +
+
+ +

◆ IMB_FEATURE_VAES

+ +
+
+ + + + +
#define IMB_FEATURE_VAES   (1ULL << 12)
+
+ +
+
+ +

◆ IMB_FEATURE_VPCLMULQDQ

+ +
+
+ + + + +
#define IMB_FEATURE_VPCLMULQDQ   (1ULL << 13)
+
+ +
+
+ +

◆ IMB_FLAG_AESNI_OFF

+ +
+
+ + + + +
#define IMB_FLAG_AESNI_OFF   (1ULL << 1)
+
+

disable use of AESNI extension

+ +
+
+ +

◆ IMB_FLAG_GFNI_OFF

+ +
+
+ + + + +
#define IMB_FLAG_GFNI_OFF   (1ULL << 2)
+
+

disable use of GFNI extension

+ +
+
+ +

◆ IMB_FLAG_SHANI_OFF

+ +
+
+ + + + +
#define IMB_FLAG_SHANI_OFF   (1ULL << 0)
+
+

disable use of SHANI extension

+ +
+
+ +

◆ IMB_FLUSH_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_FLUSH_BURST( _mgr,
 _max_jobs,
 _jobs 
)    ((_mgr)->flush_burst((_mgr), (_max_jobs), (_jobs)))
+
+ +

Force up to max_jobs outstanding jobs to completion.

+
Parameters
+ + + + +
[in,out]_mgrPointer to initialized IMB_MGR structure
[in]_max_jobsMaximum number of jobs to flush
[out]_jobsList of pointers to completed jobs
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_FLUSH_JOB

+ +
+
+ + + + + + + + +
#define IMB_FLUSH_JOB( _mgr)   ((_mgr)->flush_job((_mgr)))
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ IMB_GCM_128_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_128_KEY_LEN   (16)
+
+ +
+
+ +

◆ IMB_GCM_192_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_192_KEY_LEN   (24)
+
+ +
+
+ +

◆ IMB_GCM_256_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_256_KEY_LEN   (32)
+
+ +
+
+ +

◆ IMB_GCM_BLOCK_LEN

+ +
+
+ + + + +
#define IMB_GCM_BLOCK_LEN   16
+
+ +
+
+ +

◆ IMB_GCM_ENC_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_ENC_KEY_LEN   16
+
+ +
+
+ +

◆ IMB_GCM_IV_DATA_LEN

+ +
+
+ + + + +
#define IMB_GCM_IV_DATA_LEN   (12)
+
+

IV data is limited to 16 bytes as follows: 12 bytes is provided by an application - pre-counter block j0: 4 byte salt (from Security Association) concatenated with 8 byte Initialization Vector (from IPSec ESP Payload). 4 byte value 0x00000001 is padded automatically by the library - there is no need to add these 4 bytes on application side anymore.

+ +
+
+ +

◆ IMB_GCM_KEY_SETS

+ +
+
+ + + + +
#define IMB_GCM_KEY_SETS   (15)
+
+

exp key + 14 exp round keys

+ +
+
+ +

◆ IMB_GET_COMPLETED_JOB

+ +
+
+ + + + + + + + +
#define IMB_GET_COMPLETED_JOB( _mgr)   ((_mgr)->get_completed_job((_mgr)))
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ IMB_GET_NEXT_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_GET_NEXT_BURST( _mgr,
 _n_jobs,
 _jobs 
)    ((_mgr)->get_next_burst((_mgr), (_n_jobs), (_jobs)))
+
+ +

Get next available burst (list of pointers to available IMB_JOB structures).

+
Parameters
+ + + + +
[in,out]_mgrPointer to initialized IMB_MGR structure
[in]_n_jobsRequested number of burst jobs
[out]_jobsList of pointers to returned jobs
+
+
+
Returns
Number of returned jobs. May be less than number of requested jobs if not enough space in queue. IMB_FLUSH_BURST() can be used to free up space.
+ +
+
+ +

◆ IMB_GET_NEXT_JOB

+ +
+
+ + + + + + + + +
#define IMB_GET_NEXT_JOB( _mgr)   ((_mgr)->get_next_job((_mgr)))
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ IMB_GHASH

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_GHASH( _mgr,
 _exp_key,
 _src,
 _len,
 _tag,
 _tagl 
)    ((_mgr)->ghash((_exp_key), (_src), (_len), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_GHASH_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_GHASH_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->ghash_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_HEC_32

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_HEC_32( _mgr,
 _src 
)   ((_mgr)->hec_32(_src))
+
+

HEC compute functions

+ +
+
+ +

◆ IMB_HEC_64

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_HEC_64( _mgr,
 _src 
)   ((_mgr)->hec_64(_src))
+
+ +
+
+ +

◆ IMB_KASUMI_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_BLOCK_SIZE   8
+
+ +
+
+ +

◆ IMB_KASUMI_DIGEST_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_DIGEST_SIZE   4
+
+ +
+
+ +

◆ IMB_KASUMI_F8_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_1_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
+ +

Kasumi byte-level f8 operation on a single buffer.

+

This function performs kasumi f8 operation on a single buffer. The key has already been scheduled with kasumi_init_f8_key_sched(). No extra bits are modified.

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivInitialization vector
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in BYTES
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_1_BUFFER_BIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_1_BUFFER_BIT( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _offset 
)
+
+Value:
((_mgr)->f8_1_buffer_bit((_exp_key), (_iv), (_src), (_dst), (_len), \
+
(_offset)))
+
+

Kasumi bit-level f8 operation on a single buffer.

+

This function performs kasumi f8 operation on a single buffer. The key has already been scheduled with kasumi_init_f8_key_sched(). No extra bits are modified.

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivInitialization vector
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in BITS
[in]_offsetOffset in BITS from begin of input buffer
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_2_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_2_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2 
)
+
+Value:
((_mgr)->f8_2_buffer((_exp_key), (_iv1), (_iv2), (_src1), (_dst1), \
+
(_len1), (_src2), (_dst2), (_len2)))
+
+

Kasumi byte-level f8 operation in parallel on two buffers.

+

This function performs kasumi f8 operation on a two buffers. They will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1Initialization vector for buffer in1
[in]_iv2Initialization vector for buffer in2
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in BYTES of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in BYTES of input buffer 2
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_3_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_3_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _src1,
 _dst1,
 _src2,
 _dst2,
 _src3,
 _dst3,
 _len 
)
+
+Value:
((_mgr)->f8_3_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_src1), \
+
(_dst1), (_src2), (_dst2), (_src3), (_dst3), \
+
(_len)))
+
+

kasumi byte-level f8 operation in parallel on three buffers

+

This function performs kasumi f8 operation on a three buffers. They must all have the same length and they will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1Initialization vector for buffer in1
[in]_iv2Initialization vector for buffer in2
[in]_iv3Initialization vector for buffer in3
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_lenCommon length in bytes for all buffers
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_4_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_4_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _iv4,
 _src1,
 _dst1,
 _src2,
 _dst2,
 _src3,
 _dst3,
 _src4,
 _dst4,
 _len 
)
+
+Value:
((_mgr)->f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_iv4), \
+
(_src1), (_dst1), (_src2), (_dst2), \
+
(_src3), (_dst3), (_src4), (_dst4), (_len)))
+
+

kasumi byte-level f8 operation in parallel on four buffers

+

This function performs kasumi f8 operation on four buffers. They must all have the same length and they will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1Initialization vector for buffer in1
[in]_iv2Initialization vector for buffer in2
[in]_iv3Initialization vector for buffer in3
[in]_iv4Initialization vector for buffer in4
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_src4Input buffer 4
[out]_dst4Output buffer 4
[in]_lenCommon length in bytes for all buffers
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_N_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)
+
+Value:
((_mgr)->f8_n_buffer((_exp_key), (_iv), (_src), (_dst), (_len), \
+
(_count)))
+
+

Kasumi f8 operation on N buffers.

+

All input buffers can have different lengths and they will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivArray of IV values
[in]_srcArray of input buffers
[out]_dstArray of output buffers
[in]_lenArray of corresponding input buffer lengths in BITS
[in]_countNumber of input buffers
+
+
+ +
+
+ +

◆ IMB_KASUMI_F9_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F9_1_BUFFER( _mgr,
 _exp_key,
 _src,
 _len,
 _tag 
)    ((_mgr)->f9_1_buffer((_exp_key), (_src), (_len), (_tag)))
+
+ +

Kasumi bit-level f9 operation on a single buffer.

+

The first QWORD of in represents the COUNT and FRESH, the last QWORD represents the DIRECTION and PADDING. (See 3GPP TS 35.201 v10.0 section 4)

+

The key has already been scheduled with kasumi_init_f9_key_sched().

+
Parameters
+ + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_srcInput buffer
[in]_lenLength in BYTES of the data to be hashed
[out]_tagComputed digest
+
+
+ +
+
+ +

◆ IMB_KASUMI_F9_1_BUFFER_USER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F9_1_BUFFER_USER( _mgr,
 _exp_key,
 _iv,
 _src,
 _len,
 _tag,
 _dir 
)
+
+Value:
((_mgr)->f9_1_buffer_user((_exp_key), (_iv), (_src), (_len), \
+
(_tag), (_dir)))
+
+

Kasumi bit-level f9 operation on a single buffer.

+

The key has already been scheduled with kasumi_init_f9_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivInitialization vector
[in]_srcInput buffer
[in]_lenLength in BITS of the data to be hashed
[out]_tagComputed digest
[in]_dirDirection bit
+
+
+ +
+
+ +

◆ IMB_KASUMI_INIT_F8_KEY_SCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_INIT_F8_KEY_SCHED( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->kasumi_init_f8_key_sched((_key), (_exp_key)))
+
+

KASUMI F8 key schedule init function.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyConfidentiality key (expected in LE format)
[out]_exp_keyKey schedule context to be initialised
+
+
+
Returns
0 on success, -1 on failure
+ +
+
+ +

◆ IMB_KASUMI_INIT_F9_KEY_SCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_INIT_F9_KEY_SCHED( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->kasumi_init_f9_key_sched((_key), (_exp_key)))
+
+

KASUMI F9 key schedule init function.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyIntegrity key (expected in LE format)
[out]_exp_keyKey schedule context to be initialised
+
+
+
Returns
0 on success, -1 on failure
+ +
+
+ +

◆ IMB_KASUMI_IV_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_IV_SIZE   8
+
+ +
+
+ +

◆ IMB_KASUMI_KEY_SCHED_SIZE

+ +
+
+ + + + + + + + +
#define IMB_KASUMI_KEY_SCHED_SIZE( _mgr)   ((_mgr)->kasumi_key_sched_size())
+
+

This function returns the size of the kasumi_key_sched_t, used to store the key schedule.

+
Parameters
+ + +
[in]_mgrPointer to multi-buffer structure
+
+
+
Returns
size of kasumi_key_sched_t type success
+ +
+
+ +

◆ IMB_KASUMI_KEY_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_KEY_SIZE   16
+
+ +
+
+ +

◆ IMB_MAX_BURST_SIZE

+ +
+
+ + + + +
#define IMB_MAX_BURST_SIZE   128
+
+ +
+
+ +

◆ IMB_MAX_JOBS

+ +
+
+ + + + +
#define IMB_MAX_JOBS   (IMB_MAX_BURST_SIZE * 2)
+
+ +
+
+ +

◆ IMB_MAX_TAG_LEN

+ +
+
+ + + + +
#define IMB_MAX_TAG_LEN   (16)
+
+

Authenticated Tag Length in bytes. Valid values are 16 (most likely), 12 or 8.

+ +
+
+ +

◆ IMB_MD5_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_MD5_BLOCK_SIZE   64
+
+ +
+
+ +

◆ IMB_MD5_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_MD5_DIGEST_SIZE_IN_BYTES   16
+
+ +
+
+ +

◆ IMB_MD5_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_MD5_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->md5_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (16 bytes)
+
+
+ +
+
+ +

◆ IMB_QUEUE_SIZE

+ +
+
+ + + + + + + + +
#define IMB_QUEUE_SIZE( _mgr)   ((_mgr)->queue_size((_mgr)))
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ IMB_SHA1

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA1( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha1((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA1.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (20 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA1_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA1_BLOCK_SIZE   64
+
+

512 bits is 64 byte blocks

+ +
+
+ +

◆ IMB_SHA1_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA1_DIGEST_SIZE_IN_BYTES   20
+
+ +
+
+ +

◆ IMB_SHA1_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA1_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha1_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with SHA1.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (20 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA224

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA224( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha224((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA224.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (28 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA224_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA224_DIGEST_SIZE_IN_BYTES   28
+
+ +
+
+ +

◆ IMB_SHA224_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA224_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha224_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with SHA224.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (28 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA256

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA256( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha256((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA256.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (32 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA256_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA256_DIGEST_SIZE_IN_BYTES   32
+
+ +
+
+ +

◆ IMB_SHA256_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA256_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha256_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with SHA256.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (32 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA384

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA384( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha384((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA384.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (48 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA384_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA384_DIGEST_SIZE_IN_BYTES   48
+
+ +
+
+ +

◆ IMB_SHA384_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA384_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha384_one_block((_src), (_tag)))
+
+

Authenticate 128-byte data buffer with SHA384.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src128-byte data buffer
[out]_tagDigest output (48 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA512

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA512( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha512((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA512.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (20 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA512_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA512_DIGEST_SIZE_IN_BYTES   64
+
+ +
+
+ +

◆ IMB_SHA512_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA512_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha512_one_block((_src), (_tag)))
+
+

Authenticate 128-byte data buffer with SHA512.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src128-byte data buffer
[out]_tagDigest output (64 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA_224_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA_224_BLOCK_SIZE   64
+
+

512 bits is 64 byte blocks

+ +
+
+ +

◆ IMB_SHA_256_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA_256_BLOCK_SIZE   64
+
+

512 bits is 64 byte blocks

+ +
+
+ +

◆ IMB_SHA_384_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA_384_BLOCK_SIZE   128
+
+ +
+
+ +

◆ IMB_SHA_512_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA_512_BLOCK_SIZE   128
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_1_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->snow3g_f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
+

This function performs snow3g f8 operation on a single buffer. The key has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iviv[3] = count iv[2] = (bearer << 27) | ((dir & 0x1) << 26) iv[1] = pIV[3] iv[0] = pIV[2]
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in bits of input buffer
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_1_BUFFER_BIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_1_BUFFER_BIT( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _offset 
)
+
+Value:
((_mgr)->snow3g_f8_1_buffer_bit((_exp_key), (_iv), (_src), \
+
(_dst), (_len), (_offset)))
+

This function performs snow3g f8 operation on a single buffer. The key has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iviv[3] = count iv[2] = (bearer << 27) | ((dir & 0x1) << 26) iv[1] = pIV[3] iv[0] = pIV[2]
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in bits of input buffer
[in]_offsetOffset in input/output buffer (in bits)
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_2_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_2_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2 
)
+
+Value:
((_mgr)->snow3g_f8_2_buffer((_exp_key), (_iv1), (_iv2), \
+
(_src1), (_dst1), (_len1), \
+
(_src2), (_dst2), (_len2)))
+

This function performs snow3g f8 operation on two buffers. They will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1IV to use for buffer pBufferIn1
[in]_iv2IV to use for buffer pBufferIn2
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in bytes of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in bytes of input buffer 2
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_4_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_4_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _iv4,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2,
 _src3,
 _dst3,
 _len3,
 _src4,
 _dst4,
 _len4 
)
+
+Value:
((_mgr)->snow3g_f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
(_iv4), (_src1), (_dst1), (_len1), \
+
(_src2), (_dst2), (_len2), \
+
(_src3), (_dst3), (_len3), \
+
(_src4), (_dst4), (_len4)))
+

This function performs snow3g f8 operation on four buffers. They will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1IV to use for buffer pBufferIn1
[in]_iv2IV to use for buffer pBufferIn2
[in]_iv3IV to use for buffer pBufferIn3
[in]_iv4IV to use for buffer pBufferIn4
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in bytes of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in bytes of input buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_len3Length in bytes of input buffer 3
[in]_src4Input buffer 4
[out]_dst4Output buffer 4
[in]_len4Length in bytes of input buffer 4
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_8_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_8_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _iv4,
 _iv5,
 _iv6,
 _iv7,
 _iv8,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2,
 _src3,
 _dst3,
 _len3,
 _src4,
 _dst4,
 _len4,
 _src5,
 _dst5,
 _len5,
 _src6,
 _dst6,
 _len6,
 _src7,
 _dst7,
 _len7,
 _src8,
 _dst8,
 _len8 
)
+
+Value:
((_mgr)->snow3g_f8_8_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
(_iv4), (_iv5), (_iv6), (_iv7), \
+
(_iv8), (_src1), (_dst1), (_len1), \
+
(_src2), (_dst2), (_len2), \
+
(_src3), (_dst3), (_len3), \
+
(_src4), (_dst4), (_len4), \
+
(_src5), (_dst5), (_len5), \
+
(_src6), (_dst6), (_len6), \
+
(_src7), (_dst7), (_len7), \
+
(_src8), (_dst8), (_len8)))
+

This function performs snow3g f8 operation on eight buffers. They will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1IV to use for buffer pBufferIn1
[in]_iv2IV to use for buffer pBufferIn2
[in]_iv3IV to use for buffer pBufferIn3
[in]_iv4IV to use for buffer pBufferIn4
[in]_iv5IV to use for buffer pBufferIn5
[in]_iv6IV to use for buffer pBufferIn6
[in]_iv7IV to use for buffer pBufferIn7
[in]_iv8IV to use for buffer pBufferIn8
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in bytes of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in bytes of input buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_len3Length in bytes of input buffer 3
[in]_src4Input buffer 4
[out]_dst4Output buffer 4
[in]_len4Length in bytes of input buffer 4
[in]_src5Input buffer 5
[out]_dst5Output buffer 5
[in]_len5Length in bytes of input buffer 5
[in]_src6Input buffer 6
[out]_dst6Output buffer 6
[in]_len6Length in bytes of input buffer 6
[in]_src7Input buffer 7
[out]_dst7Output buffer 7
[in]_len7Length in bytes of input buffer 7
[in]_src8Input buffer 8
[out]_dst8Output buffer 8
[in]_len8Length in bytes of input buffer 8
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_8_BUFFER_MULTIKEY

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_8_BUFFER_MULTIKEY( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len 
)
+
+Value:
((_mgr)->snow3g_f8_8_buffer_multikey((_exp_key), (_iv), (_src), (_dst),\
+
(_len)))
+

This function performs snow3g f8 operation on eight buffers. They will be processed with individual keys, which have already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyArray of 8 Contexts, where the scheduled keys are stored
[in]_ivArray of 8 IV values
[in]_srcArray of 8 input buffers
[out]_dstArray of 8 output buffers
[in]_lenArray of 8 corresponding input buffer lengths
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_N_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)
+
+Value:
((_mgr)->snow3g_f8_n_buffer((_exp_key), (_iv), (_src), \
+
(_dst), (_len), (_count)))
+

This function performs snow3g f8 operation in parallel on N buffers. All input buffers can have different lengths and they will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivArray of IV values
[in]_srcArray of input buffers
[out]_dstArray of output buffers - out[0] set to NULL on failure
[in]_lenArray of corresponding input buffer lengths
[in]_countNumber of input buffers
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_N_BUFFER_MULTIKEY

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_N_BUFFER_MULTIKEY( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)
+
+Value:
((_mgr)->snow3g_f8_n_buffer_multikey((_exp_key), (_iv), (_src), \
+
(_dst), (_len), (_count)))
+

This function performs snow3g f8 operation in parallel on N buffers. All input buffers can have different lengths. Confidentiallity keys can vary, schedules with snow3g_init_key_sched_multi().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyArray of Contexts, where the scheduled keys are stored
[in]_ivArray of IV values
[in]_srcArray of input buffers
[out]_dstArray of output buffers
    +
  • out[0] set to NULL on failure
    • +
+
[in]_lenArray of corresponding input buffer lengths
[in]_countNumber of input buffers
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F9_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F9_1_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _len,
 _tag 
)    ((_mgr)->snow3g_f9_1_buffer((_exp_key), (_iv), (_src), (_len), (_tag)))
+
+

This function performs a snow3g f9 operation on a single block of data. The key has already been scheduled with snow3g_init_f8_key_sched().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iviv[3] = _BSWAP32(fresh^(dir<<15)) iv[2] = _BSWAP32(count^(dir<<31)) iv[1] = _BSWAP32(fresh) iv[0] = _BSWAP32(count)
[in]_srcInput buffer
[in]_lenLength in bits of the data to be hashed
[out]_tagComputed digest
+
+
+ +
+
+ +

◆ IMB_SNOW3G_INIT_KEY_SCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_INIT_KEY_SCHED( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->snow3g_init_key_sched((_key), (_exp_key)))
+
+

Snow3g key schedule init function.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyConfidentiality/Integrity key (expected in LE format)
[out]_exp_keyKey schedule context to be initialised
+
+
+
Returns
0 on success
+
+-1 on error
+ +
+
+ +

◆ IMB_SNOW3G_KEY_SCHED_SIZE

+ +
+
+ + + + + + + + +
#define IMB_SNOW3G_KEY_SCHED_SIZE( _mgr)   ((_mgr)->snow3g_key_sched_size())
+
+

This function returns the size of the snow3g_key_schedule_t, used to store the key schedule.

+
Parameters
+ + +
[in]_mgrPointer to multi-buffer structure
+
+
+
Returns
size of snow3g_key_schedule_t type
+ +
+
+ +

◆ IMB_SUBMIT_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_BURST( _mgr,
 _n_jobs,
 _jobs 
)    ((_mgr)->submit_burst((_mgr), (_n_jobs), (_jobs)))
+
+ +

Submit multiple jobs to be processed after validating.

+

Prior to submission, _jobs need to be initialized with correct crypto job parameters and followed with a call to imb_set_session().

+
Parameters
+ + + + +
[in,out]_mgrPointer to initialized IMB_MGR structure
[in]_n_jobsNumber of jobs to submit for processing
[in,out]_jobsIn: List of pointers to jobs for submission Out: List of pointers to completed jobs
+
+
+
See also
imb_set_session()
+
Returns
Number of completed jobs or zero on error. If zero, imb_get_errno() can be used to check for potential error conditions and _jobs[0] contains pointer to invalid job
+ +
+
+ +

◆ IMB_SUBMIT_BURST_NOCHECK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_BURST_NOCHECK( _mgr,
 _n_jobs,
 _jobs 
)    ((_mgr)->submit_burst_nocheck((_mgr), (_n_jobs), (_jobs)))
+
+ +

Submit multiple jobs to be processed without validating.

+

Prior to submission _jobs need to be initialized with correct crypto job parameters and followed with call to imb_set_session().

+
Parameters
+ + + + +
[in,out]_mgrPointer to initialized IMB_MGR structure
[in]_n_jobsNumber of jobs to submit for processing
[in,out]_jobsIn: List of pointers to jobs for submission Out: List of pointers to completed jobs
+
+
+
See also
imb_set_session()
+
Returns
Number of completed jobs or zero on error
+ +
+
+ +

◆ IMB_SUBMIT_CIPHER_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_CIPHER_BURST( _mgr,
 _jobs,
 _n_jobs,
 _cipher,
 _dir,
 _key_size 
)
+
+Value:
((_mgr)->submit_cipher_burst((_mgr), (_jobs), (_n_jobs), \
+
(_cipher), (_dir), (_key_size)))
+

Submit multiple cipher jobs to be processed synchronously after validating.

+
Parameters
+ + + + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_cipherCipher algorithm of type IMB_CIPHER_MODE
[in]_dirCipher direction of type IMB_CIPHER_DIRECTION
[in]_key_sizeKey size in bytes of type IMB_KEY_SIZE_BYTES
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_CIPHER_BURST_NOCHECK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_CIPHER_BURST_NOCHECK( _mgr,
 _jobs,
 _n_jobs,
 _cipher,
 _dir,
 _key_size 
)
+
+Value:
((_mgr)->submit_cipher_burst_nocheck((_mgr), (_jobs), (_n_jobs),\
+
(_cipher), (_dir), (_key_size)))
+

Submit multiple cipher jobs to be processed synchronously without validating.

+

This is more performant but less secure than IMB_SUBMIT_CIPHER_BURST().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_cipherCipher algorithm of type IMB_CIPHER_MODE
[in]_dirCipher direction of type IMB_CIPHER_DIRECTION
[in]_key_sizeKey size in bytes of type IMB_KEY_SIZE_BYTES
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_HASH_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_HASH_BURST( _mgr,
 _jobs,
 _n_jobs,
 _hash 
)    ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash)))
+
+

Submit multiple hash jobs to be processed synchronously after validating.

+
Parameters
+ + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_hashHash algorithm of type IMB_HASH_ALG
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_HASH_BURST_NOCHECK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_HASH_BURST_NOCHECK( _mgr,
 _jobs,
 _n_jobs,
 _hash 
)    ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash)))
+
+

Submit multiple hash jobs to be processed synchronously without validating.

+

This is more performant but less secure than IMB_SUBMIT_HASH_BURST().

+
Parameters
+ + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_hashHash algorithm of type IMB_HASH_ALG
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_JOB

+ +
+
+ + + + + + + + +
#define IMB_SUBMIT_JOB( _mgr)   ((_mgr)->submit_job((_mgr)))
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ IMB_SUBMIT_JOB_NOCHECK

+ +
+
+ + + + + + + + +
#define IMB_SUBMIT_JOB_NOCHECK( _mgr)   ((_mgr)->submit_job_nocheck((_mgr)))
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ IMB_VERSION

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_VERSION( a,
 b,
 c 
)   (((a) << 16) + ((b) << 8) + (c))
+
+

Macro to translate version number

+ +
+
+ +

◆ IMB_VERSION_NUM

+ +
+
+ + + + +
#define IMB_VERSION_NUM   0x10400
+
+ +
+
+ +

◆ IMB_VERSION_STR

+ +
+
+ + + + +
#define IMB_VERSION_STR   "1.4.0-dev"
+
+

Library version

+ +
+
+ +

◆ IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX

+ +
+
+ + + + +
#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX   16
+
+ +
+
+ +

◆ IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN

+ +
+
+ + + + +
#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN   4
+
+ +
+
+ +

◆ IMB_ZUC256_IV_LEN_IN_BYTES_MAX

+ +
+
+ + + + +
#define IMB_ZUC256_IV_LEN_IN_BYTES_MAX   25
+
+ +
+
+ +

◆ IMB_ZUC256_IV_LEN_IN_BYTES_MIN

+ +
+
+ + + + +
#define IMB_ZUC256_IV_LEN_IN_BYTES_MIN   23
+
+ +
+
+ +

◆ IMB_ZUC256_KEY_LEN_IN_BYTES

+ +
+
+ + + + +
#define IMB_ZUC256_KEY_LEN_IN_BYTES   32
+
+ +
+
+ +

◆ IMB_ZUC_DIGEST_LEN_IN_BYTES

+ +
+
+ + + + +
#define IMB_ZUC_DIGEST_LEN_IN_BYTES   4
+
+ +
+
+ +

◆ IMB_ZUC_EEA3_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EEA3_1_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->eea3_1_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
+ +

ZUC EEA3 Confidentiality functions.

+
Parameters
+ + + + + + + +
_mgrPointer to multi-buffer structure
_keyPointer to key
_ivPointer to 16-byte IV
_srcPointer to Plaintext/Ciphertext input.
_dstPointer to Ciphertext/Plaintext output.
_lenLength of input data in bytes.
+
+
+ +
+
+ +

◆ IMB_ZUC_EEA3_4_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EEA3_4_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->eea3_4_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
+ +
+
+ +

◆ IMB_ZUC_EEA3_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EEA3_N_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)    ((_mgr)->eea3_n_buffer((_key), (_iv), (_src), (_dst), (_len), (_count)))
+
+ +
+
+ +

◆ IMB_ZUC_EIA3_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EIA3_1_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _len,
 _tag 
)    ((_mgr)->eia3_1_buffer((_key), (_iv), (_src), (_len), (_tag)))
+
+ +

ZUC EIA3 Integrity function.

+
Parameters
+ + + + + + + +
_mgrPointer to multi-buffer structure
_keyPointer to key
_ivPointer to 16-byte IV
_srcPointer to Plaintext/Ciphertext input.
_lenLength of input data in bits.
_tagPointer to Authenticated Tag output (4 bytes)
+
+
+ +
+
+ +

◆ IMB_ZUC_EIA3_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EIA3_N_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _len,
 _tag,
 _count 
)    ((_mgr)->eia3_n_buffer((_key), (_iv), (_src), (_len), (_tag), (_count)))
+
+ +
+
+ +

◆ IMB_ZUC_IV_LEN_IN_BYTES

+ +
+
+ + + + +
#define IMB_ZUC_IV_LEN_IN_BYTES   16
+
+ +
+
+ +

◆ IMB_ZUC_KEY_LEN_IN_BYTES

+ +
+
+ + + + +
#define IMB_ZUC_KEY_LEN_IN_BYTES   16
+
+ +
+
+ +

◆ KASUMI_KEY_SCHEDULE_SIZE

+ +
+
+ + + + +
#define KASUMI_KEY_SCHEDULE_SIZE   64
+
+ +
+
+

Typedef Documentation

+ +

◆ aes_cfb_t

+ +
+
+ + + + +
typedef void(* aes_cfb_t) (void *, const void *, const void *, const void *, uint64_t)
+
+ +
+
+ +

◆ aes_ecb_quic_t

+ +
+
+ + + + +
typedef void(* aes_ecb_quic_t) (const void *, const void *, void *out, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_finalize_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_iv_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, const uint64_t, const uint8_t *, uint8_t const *, const uint64_t, uint8_t *, const uint64_t, const uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, uint64_t, const uint8_t *, uint8_t const *, uint64_t, uint8_t *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_update_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_update_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint8_t *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_init_t

+ +
+
+ + + + +
typedef void(* aes_gcm_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, uint8_t const *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_init_var_iv_t

+ +
+
+ + + + +
typedef void(* aes_gcm_init_var_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t, const uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ aes_gcm_pre_t

+ +
+
+ + + + +
typedef void(* aes_gcm_pre_t) (const void *, struct gcm_key_data *)
+
+ +
+
+ +

◆ aes_gcm_precomp_t

+ +
+
+ + + + +
typedef void(* aes_gcm_precomp_t) (struct gcm_key_data *)
+
+ +
+
+ +

◆ aes_gmac_finalize_t

+ +
+
+ + + + +
typedef void(* aes_gmac_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ aes_gmac_init_t

+ +
+
+ + + + +
typedef void(* aes_gmac_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ aes_gmac_update_t

+ +
+
+ + + + +
typedef void(* aes_gmac_update_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ burst_fn_t

+ +
+
+ + + + +
typedef uint32_t(* burst_fn_t) (struct IMB_MGR *, const uint32_t, struct IMB_JOB **)
+
+ +
+
+ +

◆ chacha_poly_enc_dec_update_t

+ +
+
+ + + + +
typedef void(* chacha_poly_enc_dec_update_t) (const void *, struct chacha20_poly1305_context_data *, void *, const void *, const uint64_t)
+
+ +
+
+ +

◆ chacha_poly_finalize_t

+ +
+
+ + + + +
typedef void(* chacha_poly_finalize_t) (struct chacha20_poly1305_context_data *, void *, const uint64_t)
+
+ +
+
+ +

◆ chacha_poly_init_t

+ +
+
+ + + + +
typedef void(* chacha_poly_init_t) (const void *, struct chacha20_poly1305_context_data *, const void *, const void *, const uint64_t)
+
+ +
+
+ +

◆ cmac_subkey_gen_t

+ +
+
+ + + + +
typedef void(* cmac_subkey_gen_t) (const void *, void *, void *)
+
+ +
+
+ +

◆ crc32_fn_t

+ +
+
+ + + + +
typedef uint32_t(* crc32_fn_t) (const void *, const uint64_t)
+
+ +
+
+ +

◆ des_keysched_t

+ +
+
+ + + + +
typedef int(* des_keysched_t) (uint64_t *, const void *)
+
+ +
+
+ +

◆ flush_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* flush_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ get_completed_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* get_completed_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ get_next_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* get_next_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ ghash_t

+ +
+
+ + + + +
typedef void(* ghash_t) (const struct gcm_key_data *, const void *, const uint64_t, void *, const uint64_t)
+
+ +
+
+ +

◆ hash_fn_t

+ +
+
+ + + + +
typedef void(* hash_fn_t) (const void *, const uint64_t, void *)
+
+ +
+
+ +

◆ hash_one_block_t

+ +
+
+ + + + +
typedef void(* hash_one_block_t) (const void *, void *)
+
+ +
+
+ +

◆ hec_32_t

+ +
+
+ + + + +
typedef uint32_t(* hec_32_t) (const uint8_t *)
+
+ +
+
+ +

◆ hec_64_t

+ +
+
+ + + + +
typedef uint64_t(* hec_64_t) (const uint8_t *)
+
+ +
+
+ +

◆ IMB_JOB

+ +
+
+ + + + +
typedef struct IMB_JOB IMB_JOB
+
+

Job structure.

+

For AES, enc_keys and dec_keys are expected to point to expanded keys structure.

    +
  • AES-CTR, AES-ECB and AES-CCM, only enc_keys is used
    • +
  • DOCSIS (AES-CBC + AES-CFB), both pointers are used enc_keys has to be set always for the partial block
    • +
+

For DES, enc_keys and dec_keys are expected to point to DES key schedule.

    +
  • same key schedule used for enc and dec operations
    • +
+

For 3DES, enc_keys and dec_keys are expected to point to an array of 3 pointers for the corresponding 3 key schedules.

    +
  • same key schedule used for enc and dec operations
    • +
+ +
+
+ +

◆ IMB_MGR

+ +
+
+ + + + +
typedef struct IMB_MGR IMB_MGR
+
+ +
+
+ +

◆ init_mb_mgr_t

+ +
+
+ + + + +
typedef void(* init_mb_mgr_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ kasumi_f8_1_buffer_bit_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_1_buffer_bit_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_1_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_1_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_2_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_2_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_3_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_3_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_4_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_4_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_n_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_n_buffer_t) (const kasumi_key_sched_t *, const uint64_t *, const void *const *, void **, const uint32_t *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f9_1_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f9_1_buffer_t) (const kasumi_key_sched_t *, const void *, const uint32_t, void *)
+
+ +
+
+ +

◆ kasumi_f9_1_buffer_user_t

+ +
+
+ + + + +
typedef void(* kasumi_f9_1_buffer_user_t) (const kasumi_key_sched_t *, const uint64_t, const void *, const uint32_t, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_init_f8_key_sched_t

+ +
+
+ + + + +
typedef int(* kasumi_init_f8_key_sched_t) (const void *, kasumi_key_sched_t *)
+
+ +
+
+ +

◆ kasumi_init_f9_key_sched_t

+ +
+
+ + + + +
typedef int(* kasumi_init_f9_key_sched_t) (const void *, kasumi_key_sched_t *)
+
+ +
+
+ +

◆ kasumi_key_sched_size_t

+ +
+
+ + + + +
typedef size_t(* kasumi_key_sched_size_t) (void)
+
+ +
+
+ +

◆ kasumi_key_sched_t

+ +
+
+ + + + +
typedef struct kasumi_key_sched_s kasumi_key_sched_t
+
+

Structure to maintain internal key scheduling

+ +
+
+ +

◆ keyexp_t

+ +
+
+ + + + +
typedef void(* keyexp_t) (const void *, void *, void *)
+
+ +
+
+ +

◆ queue_size_t

+ +
+
+ + + + +
typedef uint32_t(* queue_size_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ snow3g_f8_1_buffer_bit_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_1_buffer_bit_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_1_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_2_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_2_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_4_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_4_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_8_buffer_multikey_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_8_buffer_multikey_t) (const snow3g_key_schedule_t *const[], const void *const[], const void *const[], void *[], const uint32_t[])
+
+ +
+
+ +

◆ snow3g_f8_8_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_8_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_n_buffer_multikey_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_n_buffer_multikey_t) (const snow3g_key_schedule_t *const[], const void *const[], const void *const[], void *[], const uint32_t[], const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_n_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_n_buffer_t) (const snow3g_key_schedule_t *, const void *const[], const void *const[], void *[], const uint32_t[], const uint32_t)
+
+ +
+
+ +

◆ snow3g_f9_1_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f9_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const uint64_t, void *)
+
+ +
+
+ +

◆ snow3g_init_key_sched_t

+ +
+
+ + + + +
typedef int(* snow3g_init_key_sched_t) (const void *, snow3g_key_schedule_t *)
+
+ +
+
+ +

◆ snow3g_key_sched_size_t

+ +
+
+ + + + +
typedef size_t(* snow3g_key_sched_size_t) (void)
+
+ +
+
+ +

◆ snow3g_key_schedule_t

+ +
+
+ + + + +
typedef struct snow3g_key_schedule_s snow3g_key_schedule_t
+
+

Snow3G key scheduling structure

+ +
+
+ +

◆ submit_cipher_burst_t

+ +
+
+ + + + +
typedef uint32_t(* submit_cipher_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size)
+
+ +
+
+ +

◆ submit_hash_burst_t

+ +
+
+ + + + +
typedef uint32_t(* submit_hash_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_HASH_ALG hash)
+
+ +
+
+ +

◆ submit_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* submit_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ xcbc_keyexp_t

+ +
+
+ + + + +
typedef void(* xcbc_keyexp_t) (const void *, void *, void *, void *)
+
+ +
+
+ +

◆ zuc_eea3_1_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eea3_1_buffer_t) (const void *, const void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ zuc_eea3_4_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eea3_4_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *)
+
+ +
+
+ +

◆ zuc_eea3_n_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eea3_n_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *, const uint32_t)
+
+ +
+
+ +

◆ zuc_eia3_1_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eia3_1_buffer_t) (const void *, const void *, const void *, const uint32_t, uint32_t *)
+
+ +
+
+ +

◆ zuc_eia3_n_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eia3_n_buffer_t) (const void *const *, const void *const *, const void *const *, const uint32_t *, uint32_t **, const uint32_t)
+
+ +
+
+

Enumeration Type Documentation

+ +

◆ IMB_ARCH

+ +
+
+ + + + +
enum IMB_ARCH
+
+

Architecture definitions

+ + + + + + + + +
Enumerator
IMB_ARCH_NONE 
IMB_ARCH_NOAESNI 
IMB_ARCH_SSE 
IMB_ARCH_AVX 
IMB_ARCH_AVX2 
IMB_ARCH_AVX512 
IMB_ARCH_NUM 
+ +
+
+ +

◆ IMB_CHAIN_ORDER

+ +
+
+ + + + +
enum IMB_CHAIN_ORDER
+
+ + + +
Enumerator
IMB_ORDER_CIPHER_HASH 
IMB_ORDER_HASH_CIPHER 
+ +
+
+ +

◆ IMB_CIPHER_DIRECTION

+ +
+
+ + + + +
enum IMB_CIPHER_DIRECTION
+
+ + + +
Enumerator
IMB_DIR_ENCRYPT 
IMB_DIR_DECRYPT 
+ +
+
+ +

◆ IMB_CIPHER_MODE

+ +
+
+ + + + +
enum IMB_CIPHER_MODE
+
+

IMB_ERR_MIN should be higher than __ELASTERROR to avoid overlap with standard error values

+ + + + + + + + + + + + + + + + + + + + + + + + + +
Enumerator
IMB_CIPHER_CBC 
IMB_CIPHER_CNTR 
IMB_CIPHER_NULL 
IMB_CIPHER_DOCSIS_SEC_BPI 
IMB_CIPHER_GCM 
IMB_CIPHER_CUSTOM 
IMB_CIPHER_DES 
IMB_CIPHER_DOCSIS_DES 
IMB_CIPHER_CCM 
IMB_CIPHER_DES3 
IMB_CIPHER_PON_AES_CNTR 
IMB_CIPHER_ECB 
IMB_CIPHER_CNTR_BITLEN 

128-EEA2/NEA2 (3GPP)

+
IMB_CIPHER_ZUC_EEA3 

128-EEA3/NEA3 (3GPP)

+
IMB_CIPHER_SNOW3G_UEA2_BITLEN 

128-UEA2 (3GPP)

+
IMB_CIPHER_KASUMI_UEA1_BITLEN 

128-UEA1 (3GPP)

+
IMB_CIPHER_CBCS_1_9 

MPEG CENC (ISO 23001-7)

+
IMB_CIPHER_CHACHA20 
IMB_CIPHER_CHACHA20_POLY1305 

AEAD CHACHA20

+
IMB_CIPHER_CHACHA20_POLY1305_SGL 

AEAD CHACHA20 with SGL support

+
IMB_CIPHER_SNOW_V 
IMB_CIPHER_SNOW_V_AEAD 
IMB_CIPHER_GCM_SGL 
IMB_CIPHER_NUM 
+ +
+
+ +

◆ IMB_ERR

+ +
+
+ + + + +
enum IMB_ERR
+
+

Library error types

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Enumerator
IMB_ERR_MIN 
IMB_ERR_NULL_MBMGR 
IMB_ERR_JOB_NULL_SRC 
IMB_ERR_JOB_NULL_DST 
IMB_ERR_JOB_NULL_KEY 
IMB_ERR_JOB_NULL_IV 
IMB_ERR_JOB_NULL_AUTH 
IMB_ERR_JOB_NULL_AAD 
IMB_ERR_JOB_CIPH_LEN 
IMB_ERR_JOB_AUTH_LEN 
IMB_ERR_JOB_IV_LEN 
IMB_ERR_JOB_KEY_LEN 
IMB_ERR_JOB_AUTH_TAG_LEN 
IMB_ERR_JOB_AAD_LEN 
IMB_ERR_JOB_SRC_OFFSET 
IMB_ERR_JOB_CHAIN_ORDER 
IMB_ERR_CIPH_MODE 
IMB_ERR_HASH_ALGO 
IMB_ERR_JOB_NULL_AUTH_KEY 
IMB_ERR_JOB_NULL_SGL_CTX 
IMB_ERR_JOB_NULL_NEXT_IV 
IMB_ERR_JOB_PON_PLI 
IMB_ERR_NULL_SRC 
IMB_ERR_NULL_DST 
IMB_ERR_NULL_KEY 
IMB_ERR_NULL_EXP_KEY 
IMB_ERR_NULL_IV 
IMB_ERR_NULL_AUTH 
IMB_ERR_NULL_AAD 
IMB_ERR_CIPH_LEN 
IMB_ERR_AUTH_LEN 
IMB_ERR_IV_LEN 
IMB_ERR_KEY_LEN 
IMB_ERR_AUTH_TAG_LEN 
IMB_ERR_AAD_LEN 
IMB_ERR_SRC_OFFSET 
IMB_ERR_NULL_AUTH_KEY 
IMB_ERR_NULL_CTX 
IMB_ERR_NO_AESNI_EMU 
IMB_ERR_JOB_NULL_HMAC_OPAD 
IMB_ERR_JOB_NULL_HMAC_IPAD 
IMB_ERR_JOB_NULL_XCBC_K1_EXP 
IMB_ERR_JOB_NULL_XCBC_K2 
IMB_ERR_JOB_NULL_XCBC_K3 
IMB_ERR_JOB_CIPH_DIR 
IMB_ERR_JOB_NULL_GHASH_INIT_TAG 
IMB_ERR_MISSING_CPUFLAGS_INIT_MGR 
IMB_ERR_NULL_JOB 
IMB_ERR_QUEUE_SPACE 
IMB_ERR_NULL_BURST 
IMB_ERR_BURST_SIZE 
IMB_ERR_BURST_OOO 
IMB_ERR_SELFTEST 
IMB_ERR_BURST_SUITE_ID 
IMB_ERR_MAX 
+ +
+
+ +

◆ IMB_HASH_ALG

+ +
+
+ + + + +
enum IMB_HASH_ALG
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Enumerator
IMB_AUTH_HMAC_SHA_1 

HMAC-SHA1

+
IMB_AUTH_HMAC_SHA_224 

HMAC-SHA224

+
IMB_AUTH_HMAC_SHA_256 

HMAC-SHA256

+
IMB_AUTH_HMAC_SHA_384 

HMAC-SHA384

+
IMB_AUTH_HMAC_SHA_512 

HMAC-SHA512

+
IMB_AUTH_AES_XCBC 
IMB_AUTH_MD5 

HMAC-MD5

+
IMB_AUTH_NULL 
IMB_AUTH_AES_GMAC 
IMB_AUTH_CUSTOM 
IMB_AUTH_AES_CCM 

AES128-CCM

+
IMB_AUTH_AES_CMAC 

AES128-CMAC

+
IMB_AUTH_SHA_1 

SHA1

+
IMB_AUTH_SHA_224 

SHA224

+
IMB_AUTH_SHA_256 

SHA256

+
IMB_AUTH_SHA_384 

SHA384

+
IMB_AUTH_SHA_512 

SHA512

+
IMB_AUTH_AES_CMAC_BITLEN 

128-EIA2/NIA2 (3GPP)

+
IMB_AUTH_PON_CRC_BIP 
IMB_AUTH_ZUC_EIA3_BITLEN 

128-EIA3/NIA3 (3GPP)

+
IMB_AUTH_DOCSIS_CRC32 

with DOCSIS_SEC_BPI only

+
IMB_AUTH_SNOW3G_UIA2_BITLEN 

128-UIA2 (3GPP)

+
IMB_AUTH_KASUMI_UIA1 

128-UIA1 (3GPP)

+
IMB_AUTH_AES_GMAC_128 

AES-GMAC (128-bit key)

+
IMB_AUTH_AES_GMAC_192 

AES-GMAC (192-bit key)

+
IMB_AUTH_AES_GMAC_256 

AES-GMAC (256-bit key)

+
IMB_AUTH_AES_CMAC_256 

AES256-CMAC

+
IMB_AUTH_POLY1305 

POLY1305

+
IMB_AUTH_CHACHA20_POLY1305 

AEAD POLY1305

+
IMB_AUTH_CHACHA20_POLY1305_SGL 

AEAD CHACHA20 with SGL support

+
IMB_AUTH_ZUC256_EIA3_BITLEN 

256-EIA3/NIA3 (3GPP)

+
IMB_AUTH_SNOW_V_AEAD 

SNOW-V-AEAD

+
IMB_AUTH_GCM_SGL 

AES-GCM with SGL support

+
IMB_AUTH_CRC32_ETHERNET_FCS 

CRC32-ETHERNET-FCS

+
IMB_AUTH_CRC32_SCTP 

CRC32-SCTP

+
IMB_AUTH_CRC32_WIMAX_OFDMA_DATA 

CRC32-WIMAX-OFDMA-DATA

+
IMB_AUTH_CRC24_LTE_A 

CRC32-LTE-A

+
IMB_AUTH_CRC24_LTE_B 

CRC32-LTE-B

+
IMB_AUTH_CRC16_X25 

CRC16-X25

+
IMB_AUTH_CRC16_FP_DATA 

CRC16-FP-DATA

+
IMB_AUTH_CRC11_FP_HEADER 

CRC11-FP-HEADER

+
IMB_AUTH_CRC10_IUUP_DATA 

CRC10-IUUP-DATA

+
IMB_AUTH_CRC8_WIMAX_OFDMA_HCS 

CRC8-WIMAX-OFDMA-HCS

+
IMB_AUTH_CRC7_FP_HEADER 

CRC7-FP-HEADER

+
IMB_AUTH_CRC6_IUUP_HEADER 

CRC6-IUUP-HEADER

+
IMB_AUTH_GHASH 

GHASH

+
IMB_AUTH_NUM 
+ +
+
+ +

◆ IMB_KEY_SIZE_BYTES

+ +
+
+ + + + +
enum IMB_KEY_SIZE_BYTES
+
+ + + + + +
Enumerator
IMB_KEY_64_BYTES 
IMB_KEY_128_BYTES 
IMB_KEY_192_BYTES 
IMB_KEY_256_BYTES 
+ +
+
+ +

◆ IMB_SGL_STATE

+ +
+
+ + + + +
enum IMB_SGL_STATE
+
+ + + + + +
Enumerator
IMB_SGL_INIT 
IMB_SGL_UPDATE 
IMB_SGL_COMPLETE 
IMB_SGL_ALL 
+ +
+
+ +

◆ IMB_STATUS

+ +
+
+ + + + +
enum IMB_STATUS
+
+

Job structure definitions

+ + + + + + + + +
Enumerator
IMB_STATUS_BEING_PROCESSED 
IMB_STATUS_COMPLETED_CIPHER 
IMB_STATUS_COMPLETED_AUTH 
IMB_STATUS_COMPLETED 

COMPLETED_CIPHER | COMPLETED_AUTH

+
IMB_STATUS_INVALID_ARGS 
IMB_STATUS_INTERNAL_ERROR 
IMB_STATUS_ERROR 
+ +
+
+

Function Documentation

+ +

◆ aes_cmac_subkey_gen_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx (const void * key_exp,
void * key1,
void * key2 
)
+
+ +

+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_cmac_subkey_gen_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2 (const void * key_exp,
void * key1,
void * key2 
)
+
+ +

+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_cmac_subkey_gen_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512 (const void * key_exp,
void * key1,
void * key2 
)
+
+ +

+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_cmac_subkey_gen_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse (const void * key_exp,
void * key1,
void * key2 
)
+
+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_init_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_128_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_192_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_256_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_pre_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-128 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-128 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_128_sse (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-128 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-192 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-192 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_192_sse (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-192 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-256 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-256 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_256_sse (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-256 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_128_avx_gen2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-128 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_128_avx_gen4()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-128 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_128_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_128_sse (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-128 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_192_avx_gen2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-192 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_192_avx_gen4()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-192 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_192_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_192_sse (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-192 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_256_avx_gen2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-256 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_256_avx_gen4()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-256 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_256_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_256_sse (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-256 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_keyexp_128_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_avx (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_avx2 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_avx512 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_sse (const void * key,
void * enc_exp_keys 
)
+
+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_sse (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_avx (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_avx2 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_avx512 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_sse (const void * key,
void * enc_exp_keys 
)
+
+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_sse (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+

Generate encryption/decryption AES-192 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
[out]dec_exp_keysAES-192 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_avx (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_avx2 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_avx512 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_sse (const void * key,
void * enc_exp_keys 
)
+
+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_sse (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+ +

+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2 (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+ +

+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512 (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+ +

+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_sse (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ alloc_mb_mgr()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_MGR * alloc_mb_mgr (uint64_t flags)
+
+ +

Allocates memory for multi-buffer manager instance.

+

get_next_job returns a job object. This must be filled in and returned via submit_job before get_next_job is called again. After submit_job is called, one should call get_completed_job() at least once (and preferably until it returns NULL). get_completed_job and flush_job returns a job object. This job object ceases to be usable at the next call to get_next_job

+

For binary compatibility between library versions it is recommended to use this API.

+
Parameters
+ + +
flagsmulti-buffer manager flags IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, currently SHANI is only available for SSE IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. IMB_FLAG_GFNI_OFF - disable use (and detection) of Galois Field extensions.
+
+
+
Returns
Pointer to allocated memory for IMB_MGR structure
+
Return values
+ + +
NULLon allocation error
+
+
+ +
+
+ +

◆ des_cfb_one()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void des_cfb_one (void * out,
const void * in,
const uint64_t * iv,
const uint64_t * ks,
const int len 
)
+
+ +

DES-CFB Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + +
[out]outPlaintext/Ciphertext output
[in]inPlaintext/Ciphertext input
[in]ivPointer to 8 byte IV
[in]ksPointer to DES key schedule
[in]lenLength of data in bytes
+
+
+ +
+
+ +

◆ des_key_schedule()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int des_key_schedule (uint64_t * ks,
const void * key 
)
+
+ +

DES key schedule set up.

+

ks buffer needs to accommodate DES_KEY_SCHED_SIZE (128) bytes of data.

+
Parameters
+ + + +
[out]ksDestination buffer to accommodate DES key schedule
[in]keyPointer to an 8 byte DES key
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
!0error
+
+
+ +
+
+ +

◆ flush_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_avx (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ flush_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_avx2 (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ flush_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_avx512 (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ flush_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_sse (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ free_mb_mgr()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void free_mb_mgr (IMB_MGR * ptr)
+
+ +

Frees memory allocated previously by alloc_mb_mgr()

+
Parameters
+ + +
[in]ptrPointer to allocated MB_MGR structure
+
+
+ +
+
+ +

◆ get_completed_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_completed_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx2 (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_completed_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx512 (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_completed_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_sse (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_next_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ get_next_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx2 (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ get_next_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx512 (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ get_next_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_sse (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ imb_clear_mem()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void imb_clear_mem (void * mem,
const size_t size 
)
+
+ +

Force clearing/zeroing of memory.

+
Parameters
+ + + +
[in]memPointer to memory address to clear
[in]sizeSize of memory to clear (in bytes)
+
+
+ +
+
+ +

◆ imb_get_errno()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT int imb_get_errno (IMB_MGR * mb_mgr)
+
+ +

API to get error status.

+
Parameters
+ + +
mb_mgrPointer to multi-buffer manager
+
+
+
Return values
+ + +
Integererror type
+
+
+ +
+
+ +

◆ imb_get_feature_flags()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint64_t imb_get_feature_flags (void )
+
+ +

Retrieves the bitmask with the features supported by the library, without having to allocate/initialize IMB_MGR;.

+
Returns
Bitmask containing feature flags
+ +
+
+ +

◆ imb_get_mb_mgr_size()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT size_t imb_get_mb_mgr_size (void )
+
+ +

Calculates necessary memory size for IMB_MGR.

+
Returns
Size for IMB_MGR (aligned to 64 bytes)
+ +
+
+ +

◆ imb_get_strerror()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT const char * imb_get_strerror (int errnum)
+
+ +

API to get description for errnum.

+
Parameters
+ + +
errnumerror type
+
+
+
Return values
+ + +
Stringdescription of errnum
+
+
+ +
+
+ +

◆ imb_get_version()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT unsigned imb_get_version (void )
+
+ +

Get library version in numerical format.

+

Use IMB_VERSION() macro to compare this numerical version against known library version.

+
Returns
library version number
+ +
+
+ +

◆ imb_get_version_str()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT const char * imb_get_version_str (void )
+
+ +

Get library version in string format.

+

API definitions

+
Returns
library version string
+ +
+
+ +

◆ imb_hmac_ipad_opad()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void imb_hmac_ipad_opad (struct IMB_MGR * mb_mgr,
const IMB_HASH_ALG sha_type,
const void * pkey,
const size_t key_len,
void * ipad_hash,
void * opad_hash 
)
+
+ +

Ipad Opad padding for HMAC.

+
Parameters
+ + + + + + + +
[in]mb_mgrPointer to initialized IMB_MGR structure
[in]sha_typeType of HMAC_SHA from IMB_HASH_ALG enum
[in]pkeyPointer to a HMAC key
[in]key_lenLength of the HMAC key
[out]ipad_hashBlock-sized inner padding
[out]opad_hashBlock-sized outer padding
+
+
+ +
+
+ +

◆ imb_quic_aes_gcm()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void imb_quic_aes_gcm (IMB_MGR * state,
const struct gcm_key_data * key_data,
const IMB_KEY_SIZE_BYTES key_size,
const IMB_CIPHER_DIRECTION cipher_dir,
void * dst_ptr_array[],
const void *const src_ptr_array[],
const uint64_t len_array[],
const void *const iv_ptr_array[],
const void *const aad_ptr_array[],
const uint64_t aad_len,
void * tag_ptr_array[],
const uint64_t tag_len,
const uint64_t num_packets 
)
+
+ +

Batch of GCM encrypt/decrypt operations with the same key.

+
Note
IV length of 12 bytes is assumed.
+
+If used out of place then AAD needs to be copied by the caller.
+
+For more info on key_data refer to IMB_AES128/192/256_GCM_PRE() API’s
+
Parameters
+ + + + + + + + + + + + + + +
[in]statepointer to IMB_MGR
[in]key_datainitialized key data (AES keys and hash keys)
[in]key_sizekey size (in bytes, see IMB_KEY_128_BYTES etc.)
[in]cipher_dircipher direction (IMB_DIR_ENCRYPT / DECRYPT)
[out]dst_ptr_arrayarray with destination pointers
[in]src_ptr_arrayarray with source pointers
[in]len_arrayarray with message lengths in bytes
[in]iv_ptr_arrayarray with IV pointers
[in]aad_ptr_arrayarray with AAD pointers
[in]aad_lenAAD length in bytes
[out]tag_ptr_arrayarray with authentication TAG pointers
[in]tag_lenauthentication TAG length in bytes
[in]num_packetsnumber of packets in this batch
+
+
+ +
+
+ +

◆ imb_quic_hp_aes_ecb()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void imb_quic_hp_aes_ecb (IMB_MGR * state,
const void * exp_key_data,
void * dst_ptr_array[],
const void *const src_ptr_array[],
const uint64_t num_packets,
const IMB_KEY_SIZE_BYTES key_size 
)
+
+ +

Batch of AES-ECB encrypt/decrypt operations with the same key.

+

Sample size is fixed to 16 bytes (read from source pointers). Mask output size is fixed to 5 bytes (written to destination pointer). Cipher direction is fixed to ENCRYPT.

+
Parameters
+ + + + + + + +
[in]statepointer to IMB_MGR
[in]exp_key_dataexpanded AES encrypt keys
[out]dst_ptr_arrayarray with destination pointers
[in]src_ptr_arrayarray with source sample pointers
[in]num_packetsnumber of packets in this batch
[in]key_sizekey size (in bytes, see IMB_KEY_128_BYTES etc.)
+
+
+ +
+
+ +

◆ imb_set_pointers_mb_mgr()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT IMB_MGR * imb_set_pointers_mb_mgr (void * ptr,
const uint64_t flags,
const unsigned reset_mgr 
)
+
+ +

Initializes IMB_MGR pointers to out-of-order managers with use of externally allocated memory.

+

imb_get_mb_mgr_size() should be called to know how much memory should be allocated externally.

+

init_mb_mgr_XXX() must be called after this function call, whereas XXX is the desired architecture.

+
Parameters
+ + + + +
[in]ptra pointer to allocated memory
[in]flagsmulti-buffer manager flags IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, currently SHANI is only available for SSE IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. IMB_FLAG_GFNI_OFF - disable use (and detection) of Galois Field extensions.
[in]reset_mgrif 0, IMB_MGR structure is not cleared, else it is.
+
+
+
Returns
Pointer to IMB_MGR structure
+ +
+
+ +

◆ imb_set_session()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT uint32_t imb_set_session (IMB_MGR * state,
IMB_JOB * job 
)
+
+ +

Sets up suite_id and session_id fields for selected cipher suite in provided job structure.

+

This is mandatory operation for BURST API as suite_id is used to speed up job dispatch process. This operation is optional but helpful for JOB API use case.

+

'session_id' field is for application use to optimize job set up process. If JOB structure provided by library for a new operation has same session ID as required for the next operation then only message pointers and sizes need to be set up by the application. All other session fields are guaranteed to be unmodified by the library:

    +
  • cipher mode
    • +
  • cipher direction
    • +
  • hash algorithm
    • +
  • key size
    • +
  • encrypt & decrypt key pointers
    • +
  • suite_id If allocated JOB structure contains different session ID then all required session and crypto operation fields need to be set up.
    • +
+

In connection oriented applications, a template filled-in job structure can be cached within connection structure and reused in submit operations.

+

For given set of parameters: cipher mode, cipher key size, cipher direction and authentication mode, suite_id field is the same.

+
See also
IMB_SUBMIT_BURST()
+
+IMB_SUBMIT_BURST_NOCHECK()
+
+IMB_SUBMIT_JOB()
+
+IMB_SUBMIT_JOB_NOCHECK()
+
Parameters
+ + + +
[in]statepointer to IMB_MGR
[in,out]jobpointer to prepared JOB structure
+
+
+
Returns
Session ID value
+
Return values
+ + +
0on error
+
+
+ +
+
+ +

◆ init_mb_mgr_auto()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_auto (IMB_MGR * state,
IMB_ARCH * arch 
)
+
+ +

Automatically initialize most performant Multi-buffer manager based on CPU features.

+
Parameters
+ + + +
[in]statePointer to MB_MGR struct
[out]archPointer to arch enum to be set (can be NULL)
+
+
+ +
+
+ +

◆ init_mb_mgr_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_avx (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ init_mb_mgr_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_avx2 (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ init_mb_mgr_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_avx512 (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ init_mb_mgr_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_sse (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ kasumi_f8_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int kasumi_f8_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of KASUMI F8 Initialization Vector.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ kasumi_f9_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int kasumi_f9_iv_gen (const uint32_t count,
const uint32_t fresh,
void * iv_ptr 
)
+
+ +

Generation of KASUMI F9 Initialization Vector.

+
Parameters
+ + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]freshFRESH (4 bytes in Little Endian)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ md5_one_block_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_avx (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ md5_one_block_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_avx2 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ md5_one_block_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_avx512 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ md5_one_block_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_sse (const void * data,
void * digest 
)
+
+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ queue_size_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_avx (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ queue_size_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_avx2 (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ queue_size_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_avx512 (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ queue_size_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_sse (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ snow3g_f8_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int snow3g_f8_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of SNOW3G F8 Initialization Vector.

+

Parameters are passed in Little Endian format and used to generate the IV in Big Endian format.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes) in Big Endian format
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ snow3g_f9_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int snow3g_f9_iv_gen (const uint32_t count,
const uint32_t fresh,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of SNOW3G F9 Initialization Vector.

+

Parameters are passed in Little Endian format and used to generate the IV in Big Endian format.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]freshFRESH (4 bytes in Little Endian)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes) in Big Endian format
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ submit_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_avx (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ submit_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_avx2 (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ submit_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_avx512 (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ submit_job_nocheck_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_nocheck_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx2 (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_nocheck_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx512 (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_nocheck_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_sse (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_sse (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ zuc_eea3_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int zuc_eea3_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of ZUC-EEA3 Initialization Vector.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ zuc_eia3_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int zuc_eia3_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of ZUC-EIA3 Initialization Vector.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+
+
+ + + + diff --git a/docs/intel-ipsec-mb_8h.js b/docs/intel-ipsec-mb_8h.js new file mode 100644 index 0000000000000000000000000000000000000000..ddf585ebb26314db39550d447284e989631d472b --- /dev/null +++ b/docs/intel-ipsec-mb_8h.js @@ -0,0 +1,624 @@ +var intel_ipsec_mb_8h = +[ + [ "imb_uint128_t", "structimb__uint128__t.html", "structimb__uint128__t" ], + [ "IMB_SGL_IOV", "structIMB__SGL__IOV.html", "structIMB__SGL__IOV" ], + [ "IMB_JOB", "structIMB__JOB.html", "structIMB__JOB" ], + [ "kasumi_key_sched_s", "structkasumi__key__sched__s.html", "structkasumi__key__sched__s" ], + [ "gcm_context_data", "structgcm__context__data.html", "structgcm__context__data" ], + [ "chacha20_poly1305_context_data", "structchacha20__poly1305__context__data.html", "structchacha20__poly1305__context__data" ], + [ "gcm_key_data", "structgcm__key__data.html", "structgcm__key__data" ], + [ "snow3g_key_schedule_s", "structsnow3g__key__schedule__s.html", "structsnow3g__key__schedule__s" ], + [ "IMB_MGR", "structIMB__MGR.html", "structIMB__MGR" ], + [ "__forceinline", "intel-ipsec-mb_8h.html#af93b819ac40799ac392e16f6a90729fd", null ], + [ "DECLARE_ALIGNED", "intel-ipsec-mb_8h.html#ad920a47ed8f57af8b2cf6bcd18a3e41c", null ], + [ "IMB_AES128_CFB_ONE", "intel-ipsec-mb_8h.html#a1e122b460a714f2981f94bf7f4bba547", null ], + [ "IMB_AES128_GCM_DEC", "intel-ipsec-mb_8h.html#a2b026b2a901c76a1512eb9c8c80f9709", null ], + [ "IMB_AES128_GCM_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a280b9231c76c525a62d0ef819abd2133", null ], + [ "IMB_AES128_GCM_DEC_UPDATE", "intel-ipsec-mb_8h.html#a2f6014cfaaa0ee5f454e2771ff6b5829", null ], + [ "IMB_AES128_GCM_ENC", "intel-ipsec-mb_8h.html#adb65fc01b63a475cca4cf532ace005b6", null ], + [ "IMB_AES128_GCM_ENC_FINALIZE", "intel-ipsec-mb_8h.html#aaf13d46f5832cf9e0cb50f7fcde020be", null ], + [ "IMB_AES128_GCM_ENC_UPDATE", "intel-ipsec-mb_8h.html#ab1f4c52b4145946e80ef698e469d67db", null ], + [ "IMB_AES128_GCM_INIT", "intel-ipsec-mb_8h.html#ad0a8ea6b81df94d066b028ca7bf656ef", null ], + [ "IMB_AES128_GCM_INIT_VAR_IV", "intel-ipsec-mb_8h.html#a04142f11a678c0419dff242a0ccbd2e6", null ], + [ "IMB_AES128_GCM_PRE", "intel-ipsec-mb_8h.html#a5a442c1fcd7d427bf78e804da32bdbd1", null ], + [ "IMB_AES128_GCM_PRECOMP", "intel-ipsec-mb_8h.html#a1c63dfec645afa671f93f4fa65a6996e", null ], + [ "IMB_AES128_GMAC_FINALIZE", "intel-ipsec-mb_8h.html#af0bd4f5841d41433339717e6b97c15ad", null ], + [ "IMB_AES128_GMAC_INIT", "intel-ipsec-mb_8h.html#af315c30a8cbd3c20735e22f462b47964", null ], + [ "IMB_AES128_GMAC_UPDATE", "intel-ipsec-mb_8h.html#af0b2ba1a0f9b099a06a1ba04380b0e2a", null ], + [ "IMB_AES192_GCM_DEC", "intel-ipsec-mb_8h.html#aedd2b84ebfbcb56ac5176e8793490047", null ], + [ "IMB_AES192_GCM_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a759bbe1b8494c3157f1a506c003b700b", null ], + [ "IMB_AES192_GCM_DEC_UPDATE", "intel-ipsec-mb_8h.html#ae25c4f893fc29f09d03a8ece5fddc82f", null ], + [ "IMB_AES192_GCM_ENC", "intel-ipsec-mb_8h.html#af7548f17077a0b187a4ecfde4e0ff5d9", null ], + [ "IMB_AES192_GCM_ENC_FINALIZE", "intel-ipsec-mb_8h.html#a1e36a361be137a5d49b0798f054403bd", null ], + [ "IMB_AES192_GCM_ENC_UPDATE", "intel-ipsec-mb_8h.html#a456f5bed8eb67603bdebfda230f76957", null ], + [ "IMB_AES192_GCM_INIT", "intel-ipsec-mb_8h.html#a8cd402fe539de3fc8ff4062c2804096f", null ], + [ "IMB_AES192_GCM_INIT_VAR_IV", "intel-ipsec-mb_8h.html#a97791c976660a673405e6fa22ef3d262", null ], + [ "IMB_AES192_GCM_PRE", "intel-ipsec-mb_8h.html#a003f16aac382baff7a6b83040e58e471", null ], + [ "IMB_AES192_GCM_PRECOMP", "intel-ipsec-mb_8h.html#ac7b3bcfd3ecbe7b2b8c7da963310b87b", null ], + [ "IMB_AES192_GMAC_FINALIZE", "intel-ipsec-mb_8h.html#a16a6f89fd4131c406d6ee7a1aeb663eb", null ], + [ "IMB_AES192_GMAC_INIT", "intel-ipsec-mb_8h.html#a25ed49507e66a6e98eab4a3d34e75231", null ], + [ "IMB_AES192_GMAC_UPDATE", "intel-ipsec-mb_8h.html#a113d0f4320e929f89786773f7d11c5c8", null ], + [ "IMB_AES256_CFB_ONE", "intel-ipsec-mb_8h.html#ab4d68ab67a7d4a471f5cb454301aac9d", null ], + [ "IMB_AES256_GCM_DEC", "intel-ipsec-mb_8h.html#a1eb77771626f83eba8482473d29ffbb1", null ], + [ "IMB_AES256_GCM_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a64d9473bc4f28c0b783baa4a31b9930e", null ], + [ "IMB_AES256_GCM_DEC_UPDATE", "intel-ipsec-mb_8h.html#a70847fd3bb5d1be79ebfa4e9104edabd", null ], + [ "IMB_AES256_GCM_ENC", "intel-ipsec-mb_8h.html#a556333acdc2db1a2540a1cd1af644ca9", null ], + [ "IMB_AES256_GCM_ENC_FINALIZE", "intel-ipsec-mb_8h.html#a1f49fe7109315eb479ac1c9969e2f877", null ], + [ "IMB_AES256_GCM_ENC_UPDATE", "intel-ipsec-mb_8h.html#a223c00c636e2552e7a3dd1a97952cc6a", null ], + [ "IMB_AES256_GCM_INIT", "intel-ipsec-mb_8h.html#a8049111340bf2694799eb4e42b0a82c3", null ], + [ "IMB_AES256_GCM_INIT_VAR_IV", "intel-ipsec-mb_8h.html#a61ddc1485781aba1969fe3ca97a22ded", null ], + [ "IMB_AES256_GCM_PRE", "intel-ipsec-mb_8h.html#a5771f5169d3f3af3a2419ccd8275a754", null ], + [ "IMB_AES256_GCM_PRECOMP", "intel-ipsec-mb_8h.html#a4fa417a748ca30a961334bf91297dfda", null ], + [ "IMB_AES256_GMAC_FINALIZE", "intel-ipsec-mb_8h.html#abfbb7469f7d0240bb3d9a0e5f368b19e", null ], + [ "IMB_AES256_GMAC_INIT", "intel-ipsec-mb_8h.html#a3d57940a54e9d94fa14dc008cd959092", null ], + [ "IMB_AES256_GMAC_UPDATE", "intel-ipsec-mb_8h.html#a91b8d98c7ffce8179ef953493a959d4b", null ], + [ "IMB_AES_BLOCK_SIZE", "intel-ipsec-mb_8h.html#ab1b5db5ac58714b6f69e754a0c4ca793", null ], + [ "IMB_AES_CMAC_SUBKEY_GEN_128", "intel-ipsec-mb_8h.html#aa1e9006a8a76ba7fb09b97816d7fdfe5", null ], + [ "IMB_AES_CMAC_SUBKEY_GEN_256", "intel-ipsec-mb_8h.html#aca7b22139c297858794c7268439e88f0", null ], + [ "IMB_AES_KEYEXP_128", "intel-ipsec-mb_8h.html#afa111339421e794c2643df2ecf86a8c0", null ], + [ "IMB_AES_KEYEXP_192", "intel-ipsec-mb_8h.html#a574d86849e2c9ca402305eb479ca558d", null ], + [ "IMB_AES_KEYEXP_256", "intel-ipsec-mb_8h.html#a003306124dae4920161ed6d4a7bf4e2f", null ], + [ "IMB_AES_XCBC_KEYEXP", "intel-ipsec-mb_8h.html#a5a5f2342f2b1e65e41084d915c3b2cda", null ], + [ "IMB_ASSERT", "intel-ipsec-mb_8h.html#af0f2f0bfad62f740bc2e22e60f48af8d", null ], + [ "IMB_CHACHA20_POLY1305_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a8f6a258b6a13952fad9c217194c716f0", null ], + [ "IMB_CHACHA20_POLY1305_DEC_UPDATE", "intel-ipsec-mb_8h.html#a4ac8fd429d9f356003522200811b0bed", null ], + [ "IMB_CHACHA20_POLY1305_ENC_FINALIZE", "intel-ipsec-mb_8h.html#a7575d2fafb2b970bd08a49a556602031", null ], + [ "IMB_CHACHA20_POLY1305_ENC_UPDATE", "intel-ipsec-mb_8h.html#a2abd7cd460b99467836188d62272cd2b", null ], + [ "IMB_CHACHA20_POLY1305_INIT", "intel-ipsec-mb_8h.html#adb8ac3238e6dfe39a10e64602a96e643", null ], + [ "IMB_CPUFLAGS_AVX", "intel-ipsec-mb_8h.html#a0329fede890d95a97423a714410e1576", null ], + [ "IMB_CPUFLAGS_AVX2", "intel-ipsec-mb_8h.html#ac0a92622a85423541536321231a6cb7d", null ], + [ "IMB_CPUFLAGS_AVX2_T2", "intel-ipsec-mb_8h.html#a18f17a6bcef83c3976ca7a9b4ca624de", null ], + [ "IMB_CPUFLAGS_AVX2_T3", "intel-ipsec-mb_8h.html#a6e356586d2d3d7e93ff1b23d4a90b344", null ], + [ "IMB_CPUFLAGS_AVX512", "intel-ipsec-mb_8h.html#a0a250932ede6ca7b69f5c763c24cf51d", null ], + [ "IMB_CPUFLAGS_AVX512_T2", "intel-ipsec-mb_8h.html#af7bdbff703949eb2cf2dd3d3a08803de", null ], + [ "IMB_CPUFLAGS_AVX_T2", "intel-ipsec-mb_8h.html#a85dab1abb0996e9ff4d4258fef3f9abd", null ], + [ "IMB_CPUFLAGS_NO_AESNI", "intel-ipsec-mb_8h.html#a6afb888ab232bab1c3ca027eb071f388", null ], + [ "IMB_CPUFLAGS_SSE", "intel-ipsec-mb_8h.html#a5654f563aaad731df22c8e8517c9154c", null ], + [ "IMB_CPUFLAGS_SSE_T2", "intel-ipsec-mb_8h.html#ae6776750b801d7e177c152569c10c546", null ], + [ "IMB_CPUFLAGS_SSE_T3", "intel-ipsec-mb_8h.html#aa1cc7ac79ca44f06e3caa18fdd6d441a", null ], + [ "IMB_CRC10_IUUP_DATA", "intel-ipsec-mb_8h.html#ac7406ab5572b4af93ff8dbbc8ee0c961", null ], + [ "IMB_CRC11_FP_HEADER", "intel-ipsec-mb_8h.html#ae35b6b1b19ef6d599d47591f6c1e1732", null ], + [ "IMB_CRC16_FP_DATA", "intel-ipsec-mb_8h.html#abe747d2365eb0393ab748c218029e3c6", null ], + [ "IMB_CRC16_X25", "intel-ipsec-mb_8h.html#a78217e8116876c0ecbe82fd3d417cc1c", null ], + [ "IMB_CRC24_LTE_A", "intel-ipsec-mb_8h.html#a6ff60117e0ff260865cd80323306479b", null ], + [ "IMB_CRC24_LTE_B", "intel-ipsec-mb_8h.html#ae07e15be0652ca8faa23c6f907d83e03", null ], + [ "IMB_CRC32_ETHERNET_FCS", "intel-ipsec-mb_8h.html#a075757c7f7b132a8f2ae13c18f36e2af", null ], + [ "IMB_CRC32_SCTP", "intel-ipsec-mb_8h.html#a86753ed15fcd8aab177d276dd4ebf169", null ], + [ "IMB_CRC32_WIMAX_OFDMA_DATA", "intel-ipsec-mb_8h.html#ab0a9f19127f3417cc78cfdf9091af33e", null ], + [ "IMB_CRC6_IUUP_HEADER", "intel-ipsec-mb_8h.html#ac7c45b0788456473390ca178275f9a34", null ], + [ "IMB_CRC7_FP_HEADER", "intel-ipsec-mb_8h.html#a23fe73b1f9d40348bc98f92f859df60b", null ], + [ "IMB_CRC8_WIMAX_OFDMA_HCS", "intel-ipsec-mb_8h.html#a0edd5fa8f3351ee214f7f2a267949f6d", null ], + [ "IMB_DES_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a95f4ece0b318cb36bdbe66089d9ef649", null ], + [ "IMB_DES_KEY_SCHED_SIZE", "intel-ipsec-mb_8h.html#a7d156b1b9d715299e08cca76e790e2b7", null ], + [ "IMB_DES_KEYSCHED", "intel-ipsec-mb_8h.html#a6fbb59f06f60f1900b591735ad691ac6", null ], + [ "IMB_DIM", "intel-ipsec-mb_8h.html#a1beec59f9694a41c22b6b099263c070d", null ], + [ "IMB_DLL_EXPORT", "intel-ipsec-mb_8h.html#a986643689f56109161597f67eb9d7b40", null ], + [ "IMB_DLL_LOCAL", "intel-ipsec-mb_8h.html#a479e22216a6be5645af96c4b1433339e", null ], + [ "IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE", "intel-ipsec-mb_8h.html#a74dca2bb8b0c87dec90f3b985c0266af", null ], + [ "IMB_DOCSIS_CRC32_TAG_SIZE", "intel-ipsec-mb_8h.html#a59c620471ea8c9577ed593127f82c758", null ], + [ "IMB_FEATURE_AESNI", "intel-ipsec-mb_8h.html#a19c147087d2ae918baa22f90fa9b3b05", null ], + [ "IMB_FEATURE_AESNI_EMU", "intel-ipsec-mb_8h.html#a421158db6d19c97a6bceb32eb4a839ed", null ], + [ "IMB_FEATURE_AVX", "intel-ipsec-mb_8h.html#a7af992d12b4f9a6e789384969a066056", null ], + [ "IMB_FEATURE_AVX2", "intel-ipsec-mb_8h.html#a225e2cd9c1445795fc62879eabad2530", null ], + [ "IMB_FEATURE_AVX512_IFMA", "intel-ipsec-mb_8h.html#a0134dfaffe12427a8f2182247c6f1f48", null ], + [ "IMB_FEATURE_AVX512_SKX", "intel-ipsec-mb_8h.html#ae6d6e3427c81b278d64f0189401c4964", null ], + [ "IMB_FEATURE_AVX512BW", "intel-ipsec-mb_8h.html#a00caae81d425acb6b946bebb25ac267f", null ], + [ "IMB_FEATURE_AVX512CD", "intel-ipsec-mb_8h.html#abbdb390b80ff1a92f31a7a9adc9beeb9", null ], + [ "IMB_FEATURE_AVX512DQ", "intel-ipsec-mb_8h.html#a8ca2c970faa0f62a3f4a6b6c18432bb1", null ], + [ "IMB_FEATURE_AVX512F", "intel-ipsec-mb_8h.html#ab673c08624d868ca0dd89e1a9ec28f00", null ], + [ "IMB_FEATURE_AVX512VL", "intel-ipsec-mb_8h.html#a803871900c08e2dc1b3f805ed4ecefd8", null ], + [ "IMB_FEATURE_AVX_IFMA", "intel-ipsec-mb_8h.html#a8a946f162ff5e363d04f8b44d2f48747", null ], + [ "IMB_FEATURE_BMI2", "intel-ipsec-mb_8h.html#a557f3de4bcc0bd7e9bd90e5e1d1b9b09", null ], + [ "IMB_FEATURE_CMOV", "intel-ipsec-mb_8h.html#a5eef394935c651931b1cf560f31c841c", null ], + [ "IMB_FEATURE_GFNI", "intel-ipsec-mb_8h.html#aa67ab4088ac2f8cadbe839101d276567", null ], + [ "IMB_FEATURE_PCLMULQDQ", "intel-ipsec-mb_8h.html#a1af54a4a862936a6793af2ff44163ba5", null ], + [ "IMB_FEATURE_SAFE_DATA", "intel-ipsec-mb_8h.html#abd58fdbe1f1ef53cf509a0b37ba8aa4b", null ], + [ "IMB_FEATURE_SAFE_PARAM", "intel-ipsec-mb_8h.html#a4e9506dc72ab4e6a2af463238ccb3080", null ], + [ "IMB_FEATURE_SELF_TEST", "intel-ipsec-mb_8h.html#a8b4181547ded3ff17859d2632c468151", null ], + [ "IMB_FEATURE_SELF_TEST_PASS", "intel-ipsec-mb_8h.html#a673b969c5f01579f6ca0a6b003cf1197", null ], + [ "IMB_FEATURE_SHANI", "intel-ipsec-mb_8h.html#acb9c15eabfb9bafeb15a06886fa91e90", null ], + [ "IMB_FEATURE_SSE4_2", "intel-ipsec-mb_8h.html#abcc243fc505aca3eac6963eff25e9b99", null ], + [ "IMB_FEATURE_VAES", "intel-ipsec-mb_8h.html#af4321e4e3c947f9b9fb7f204aaecf2c3", null ], + [ "IMB_FEATURE_VPCLMULQDQ", "intel-ipsec-mb_8h.html#a153d695ecd1af69a0d23385a50654614", null ], + [ "IMB_FLAG_AESNI_OFF", "intel-ipsec-mb_8h.html#a4c53fb8e39bbc6e3f39d0558934e5d55", null ], + [ "IMB_FLAG_GFNI_OFF", "intel-ipsec-mb_8h.html#ad3faac9d03a87621ae02c10337b4edf8", null ], + [ "IMB_FLAG_SHANI_OFF", "intel-ipsec-mb_8h.html#a3d45fcff99a9ea952b4dd6ef70e36e0c", null ], + [ "IMB_FLUSH_BURST", "intel-ipsec-mb_8h.html#afff52a9aaa1c1875e7c905e2fb29fc43", null ], + [ "IMB_FLUSH_JOB", "intel-ipsec-mb_8h.html#a4726e3683b3579b1e643134278386d72", null ], + [ "IMB_GCM_128_KEY_LEN", "intel-ipsec-mb_8h.html#ae97681c203afbd818a39fb797613a232", null ], + [ "IMB_GCM_192_KEY_LEN", "intel-ipsec-mb_8h.html#a7201a0be67f1e6f3757169b0166dd47d", null ], + [ "IMB_GCM_256_KEY_LEN", "intel-ipsec-mb_8h.html#a4c4c40ec87c2354832f5b182b22d2111", null ], + [ "IMB_GCM_BLOCK_LEN", "intel-ipsec-mb_8h.html#a59def6ed420a4ef91ce4a6562cc2c4d5", null ], + [ "IMB_GCM_ENC_KEY_LEN", "intel-ipsec-mb_8h.html#a0c6f54ab6663187fa401c766a981a41e", null ], + [ "IMB_GCM_IV_DATA_LEN", "intel-ipsec-mb_8h.html#a87884da318b3013893b725d047b53702", null ], + [ "IMB_GCM_KEY_SETS", "intel-ipsec-mb_8h.html#a73715b8c1ba918ce6d5c21749841b47b", null ], + [ "IMB_GET_COMPLETED_JOB", "intel-ipsec-mb_8h.html#a18e5a7f85573f0006448deda41014117", null ], + [ "IMB_GET_NEXT_BURST", "intel-ipsec-mb_8h.html#a6029af3f88d1908dd11a8f3e9eb9fd70", null ], + [ "IMB_GET_NEXT_JOB", "intel-ipsec-mb_8h.html#a34b6006fc39dd4ea23d3fec49237d382", null ], + [ "IMB_GHASH", "intel-ipsec-mb_8h.html#a477ad08a4c55e2c6458b7e86dae803ec", null ], + [ "IMB_GHASH_PRE", "intel-ipsec-mb_8h.html#a230f16784519d7e2b69de87871d35cb2", null ], + [ "IMB_HEC_32", "intel-ipsec-mb_8h.html#a37996cede66ee8036b1b113af5f553ac", null ], + [ "IMB_HEC_64", "intel-ipsec-mb_8h.html#aa0b62ba2c9fb07a1ad3c8105ec4374a4", null ], + [ "IMB_KASUMI_BLOCK_SIZE", "intel-ipsec-mb_8h.html#aa31ca303e03fc548199f95e5335889e1", null ], + [ "IMB_KASUMI_DIGEST_SIZE", "intel-ipsec-mb_8h.html#a56bd44d01c91469b57efa36f360a83bf", null ], + [ "IMB_KASUMI_F8_1_BUFFER", "intel-ipsec-mb_8h.html#a45ac331e1f9241e53fc35a1f88dec6ce", null ], + [ "IMB_KASUMI_F8_1_BUFFER_BIT", "intel-ipsec-mb_8h.html#ae3befb703f7dec2c0695e61542f3212c", null ], + [ "IMB_KASUMI_F8_2_BUFFER", "intel-ipsec-mb_8h.html#a884bfa72fc232c7e84043d0faee719ee", null ], + [ "IMB_KASUMI_F8_3_BUFFER", "intel-ipsec-mb_8h.html#af35e87948218048e4bbad7162208fe41", null ], + [ "IMB_KASUMI_F8_4_BUFFER", "intel-ipsec-mb_8h.html#a7ed6be7362e5ff7553fb4e7633bee76a", null ], + [ "IMB_KASUMI_F8_N_BUFFER", "intel-ipsec-mb_8h.html#a65b1afbadee23111046746a4d302bb0d", null ], + [ "IMB_KASUMI_F9_1_BUFFER", "intel-ipsec-mb_8h.html#a43c13bbdf83e2c75d6dd29d01dec4a1f", null ], + [ "IMB_KASUMI_F9_1_BUFFER_USER", "intel-ipsec-mb_8h.html#a66dcf1985c046ef619141a6706159940", null ], + [ "IMB_KASUMI_INIT_F8_KEY_SCHED", "intel-ipsec-mb_8h.html#aafcc540a047a6036adf4da0613ea5272", null ], + [ "IMB_KASUMI_INIT_F9_KEY_SCHED", "intel-ipsec-mb_8h.html#a80a71ab95fd99c8427e171e45cebbd92", null ], + [ "IMB_KASUMI_IV_SIZE", "intel-ipsec-mb_8h.html#adf7e13e43c6bb30489d0e26f11e39440", null ], + [ "IMB_KASUMI_KEY_SCHED_SIZE", "intel-ipsec-mb_8h.html#ae3812840fbe6b9b5ea5e84a6b86a7ed3", null ], + [ "IMB_KASUMI_KEY_SIZE", "intel-ipsec-mb_8h.html#a46d59512e3a7254e54018181b7a31102", null ], + [ "IMB_MAX_BURST_SIZE", "intel-ipsec-mb_8h.html#af9070daf9749490016e9b5106b338867", null ], + [ "IMB_MAX_JOBS", "intel-ipsec-mb_8h.html#ab12d22470a68b5d3852c8dd972784d97", null ], + [ "IMB_MAX_TAG_LEN", "intel-ipsec-mb_8h.html#a41087ea6dce2e096ebb02a613600c2ed", null ], + [ "IMB_MD5_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a567100ef1c32f08b9e75e1fb378171a3", null ], + [ "IMB_MD5_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a472528903f010fe2c365d1ff23565eaa", null ], + [ "IMB_MD5_ONE_BLOCK", "intel-ipsec-mb_8h.html#a52b27293e2a19809c5f7d0f2afab97a2", null ], + [ "IMB_QUEUE_SIZE", "intel-ipsec-mb_8h.html#a712f6e7ecb2a16f8e1bc96b84d188365", null ], + [ "IMB_SHA1", "intel-ipsec-mb_8h.html#ae61900d14cde2836621a1421141548a4", null ], + [ "IMB_SHA1_BLOCK_SIZE", "intel-ipsec-mb_8h.html#af19fd9d6768e33720cab4ecd9aabad91", null ], + [ "IMB_SHA1_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a8c821d8cf5520f6f46e9b835eeeb2066", null ], + [ "IMB_SHA1_ONE_BLOCK", "intel-ipsec-mb_8h.html#ac2f9695b12c32c303faa7c93c1c24c9b", null ], + [ "IMB_SHA224", "intel-ipsec-mb_8h.html#a542a40ced46bf97a678c6b44c325b360", null ], + [ "IMB_SHA224_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a836641341e86b80f67716eabdf4a64c5", null ], + [ "IMB_SHA224_ONE_BLOCK", "intel-ipsec-mb_8h.html#ab1f7eb5e9bcd0c69a70dc76747f919f6", null ], + [ "IMB_SHA256", "intel-ipsec-mb_8h.html#a37338b9f91315e5de0c4f3d19859aa7b", null ], + [ "IMB_SHA256_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a4bc9eee15ba0ce09f545af8916292731", null ], + [ "IMB_SHA256_ONE_BLOCK", "intel-ipsec-mb_8h.html#a6023b87b5d1c6dc7dc0547db540af997", null ], + [ "IMB_SHA384", "intel-ipsec-mb_8h.html#ac8052b8b996f866f27b4d443bcda0d5f", null ], + [ "IMB_SHA384_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a2538a5e82141eb5a54f78660241e186c", null ], + [ "IMB_SHA384_ONE_BLOCK", "intel-ipsec-mb_8h.html#a763d0cfb16263906e44f46696b16124b", null ], + [ "IMB_SHA512", "intel-ipsec-mb_8h.html#ac806d6b629633e801409d3e363d83f6c", null ], + [ "IMB_SHA512_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a9efcf370f0db0b21f7cf4a3b72eec53e", null ], + [ "IMB_SHA512_ONE_BLOCK", "intel-ipsec-mb_8h.html#a71d0a529b355a434efeeda2e01870ba9", null ], + [ "IMB_SHA_224_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a7c912b7543122de6f05355fa4848eb83", null ], + [ "IMB_SHA_256_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a7eed0159b050f8d78cddcc78b4b941ef", null ], + [ "IMB_SHA_384_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a256cc0f9eaba38030ca3f7589e479855", null ], + [ "IMB_SHA_512_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a4141433b17ade98e14a585e0409cb62d", null ], + [ "IMB_SNOW3G_F8_1_BUFFER", "intel-ipsec-mb_8h.html#a4bcfbb70da38433f8bcd3468779026f1", null ], + [ "IMB_SNOW3G_F8_1_BUFFER_BIT", "intel-ipsec-mb_8h.html#a5d7a9e694b932b1c198782e22dc1ead5", null ], + [ "IMB_SNOW3G_F8_2_BUFFER", "intel-ipsec-mb_8h.html#a51085837f0775feae26fe3885f7459b7", null ], + [ "IMB_SNOW3G_F8_4_BUFFER", "intel-ipsec-mb_8h.html#a79b873457a74ad6c50e689709d79a0fb", null ], + [ "IMB_SNOW3G_F8_8_BUFFER", "intel-ipsec-mb_8h.html#a8536e68d053999123eb104762ef51932", null ], + [ "IMB_SNOW3G_F8_8_BUFFER_MULTIKEY", "intel-ipsec-mb_8h.html#aca8fb428a3b291950cd9dc6277ad801a", null ], + [ "IMB_SNOW3G_F8_N_BUFFER", "intel-ipsec-mb_8h.html#a4538b34eaf3a3d402041ada1e8d03e2c", null ], + [ "IMB_SNOW3G_F8_N_BUFFER_MULTIKEY", "intel-ipsec-mb_8h.html#a4a5ae1b04c7aee099c2f8fcbd488c38c", null ], + [ "IMB_SNOW3G_F9_1_BUFFER", "intel-ipsec-mb_8h.html#aa6a951033156fc06c25afb4f79de8851", null ], + [ "IMB_SNOW3G_INIT_KEY_SCHED", "intel-ipsec-mb_8h.html#ac5b5ff1cc9cf44ea8587a7d054b7459d", null ], + [ "IMB_SNOW3G_KEY_SCHED_SIZE", "intel-ipsec-mb_8h.html#a1ddd766994cb5d0b8e9ef5cd580cdcd2", null ], + [ "IMB_SUBMIT_BURST", "intel-ipsec-mb_8h.html#aabfad78b032b6b160e3ba51477132524", null ], + [ "IMB_SUBMIT_BURST_NOCHECK", "intel-ipsec-mb_8h.html#aa06f0e18794d4a95d16e61dfb1af70c4", null ], + [ "IMB_SUBMIT_CIPHER_BURST", "intel-ipsec-mb_8h.html#ae487720f5112a3ecec659ab4aeada77e", null ], + [ "IMB_SUBMIT_CIPHER_BURST_NOCHECK", "intel-ipsec-mb_8h.html#af819e81f584b3b56e0201a7174b3c0eb", null ], + [ "IMB_SUBMIT_HASH_BURST", "intel-ipsec-mb_8h.html#a3702b37004e38dd27a9b8a008a0c48ed", null ], + [ "IMB_SUBMIT_HASH_BURST_NOCHECK", "intel-ipsec-mb_8h.html#afd6b80655a6129fe063d2ffb391e3015", null ], + [ "IMB_SUBMIT_JOB", "intel-ipsec-mb_8h.html#a3602a4ccaf1da4bf6787169370cef04f", null ], + [ "IMB_SUBMIT_JOB_NOCHECK", "intel-ipsec-mb_8h.html#af23c64e466272c5ae90fcf78fbcd0c3d", null ], + [ "IMB_VERSION", "intel-ipsec-mb_8h.html#ad9bcb21181d1e07b4f66d8bf0514c1a3", null ], + [ "IMB_VERSION_NUM", "intel-ipsec-mb_8h.html#a415abfc6067c0b1663a508993a4eb0d4", null ], + [ "IMB_VERSION_STR", "intel-ipsec-mb_8h.html#a874972638400dafdb542319d61bfbfbc", null ], + [ "IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX", "intel-ipsec-mb_8h.html#a2b1faea7079fa3ddb2cabeb41aeb2a0a", null ], + [ "IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN", "intel-ipsec-mb_8h.html#a3dc24506ef16ad090414608c762b3d3c", null ], + [ "IMB_ZUC256_IV_LEN_IN_BYTES_MAX", "intel-ipsec-mb_8h.html#a5ab93cc1831b961f3d04ff913f63b832", null ], + [ "IMB_ZUC256_IV_LEN_IN_BYTES_MIN", "intel-ipsec-mb_8h.html#a18e1733291bb917240ef117f52c6aa6c", null ], + [ "IMB_ZUC256_KEY_LEN_IN_BYTES", "intel-ipsec-mb_8h.html#a35d04f4dc107531e37fcab1fd5efbf93", null ], + [ "IMB_ZUC_DIGEST_LEN_IN_BYTES", "intel-ipsec-mb_8h.html#adbb9f3bfdc1d416c20a579eb3a8b0b8f", null ], + [ "IMB_ZUC_EEA3_1_BUFFER", "intel-ipsec-mb_8h.html#adb23f957dde69fb3bb4c6798d1a69da4", null ], + [ "IMB_ZUC_EEA3_4_BUFFER", "intel-ipsec-mb_8h.html#a57445dd863838d04cf44f4d79c4dbcaf", null ], + [ "IMB_ZUC_EEA3_N_BUFFER", "intel-ipsec-mb_8h.html#a6382e3185f31e61f995ab90110c52991", null ], + [ "IMB_ZUC_EIA3_1_BUFFER", "intel-ipsec-mb_8h.html#ac789fe54f3a8f92144831a67b14501d4", null ], + [ "IMB_ZUC_EIA3_N_BUFFER", "intel-ipsec-mb_8h.html#ae840d62f1246c0c4270c3319ef356d3c", null ], + [ "IMB_ZUC_IV_LEN_IN_BYTES", "intel-ipsec-mb_8h.html#aaacb0749fd75d1133414685b5694652d", null ], + [ "IMB_ZUC_KEY_LEN_IN_BYTES", "intel-ipsec-mb_8h.html#a053b0646f055d7ee3be974bb2bfe534b", null ], + [ "KASUMI_KEY_SCHEDULE_SIZE", "intel-ipsec-mb_8h.html#a071e04977525b67c8d1accc60937eb67", null ], + [ "aes_cfb_t", "intel-ipsec-mb_8h.html#aeb837fda81cdf56ffcfa4e68b5877a35", null ], + [ "aes_ecb_quic_t", "intel-ipsec-mb_8h.html#ad907e0caa52fcb257275464179620643", null ], + [ "aes_gcm_enc_dec_finalize_t", "intel-ipsec-mb_8h.html#a8c85430db7a8b695a2836dcb96653bc5", null ], + [ "aes_gcm_enc_dec_iv_t", "intel-ipsec-mb_8h.html#ac5bfb31d18fdc55451cd325c4af032c7", null ], + [ "aes_gcm_enc_dec_t", "intel-ipsec-mb_8h.html#a71fb9f25bcab533a98f3d1c5cc2c7f48", null ], + [ "aes_gcm_enc_dec_update_t", "intel-ipsec-mb_8h.html#af701adcd98fd95084d76e0cb0f926fef", null ], + [ "aes_gcm_init_t", "intel-ipsec-mb_8h.html#adeecd4bac067493d8465e8d7b6b7dd6d", null ], + [ "aes_gcm_init_var_iv_t", "intel-ipsec-mb_8h.html#a65f909001359ad986640155f8756a11d", null ], + [ "aes_gcm_pre_t", "intel-ipsec-mb_8h.html#a11e10a3ac7a08801dc68949fb0f428af", null ], + [ "aes_gcm_precomp_t", "intel-ipsec-mb_8h.html#ab50906d68121cb8e6edf2a3493a60c5c", null ], + [ "aes_gmac_finalize_t", "intel-ipsec-mb_8h.html#af3f75897a0ec8b3f22c4e2b01f7f48b7", null ], + [ "aes_gmac_init_t", "intel-ipsec-mb_8h.html#a31987251c968435d720e8656ae4621a0", null ], + [ "aes_gmac_update_t", "intel-ipsec-mb_8h.html#aba0ac59064802c6d0a182a5164568c18", null ], + [ "burst_fn_t", "intel-ipsec-mb_8h.html#aaeed8e5ec28aba6b8deef269b7c4bb28", null ], + [ "chacha_poly_enc_dec_update_t", "intel-ipsec-mb_8h.html#a437de868b8dba4f9b878d99a82f80548", null ], + [ "chacha_poly_finalize_t", "intel-ipsec-mb_8h.html#a1a96f063bf1bd07de158985b57cff4a0", null ], + [ "chacha_poly_init_t", "intel-ipsec-mb_8h.html#a4b0b9ffa331f8d675d24911639f0374f", null ], + [ "cmac_subkey_gen_t", "intel-ipsec-mb_8h.html#aab07996d622d682adc1db8286e633cb3", null ], + [ "crc32_fn_t", "intel-ipsec-mb_8h.html#a56dd197c301ad27dc9ec0862721cb73d", null ], + [ "des_keysched_t", "intel-ipsec-mb_8h.html#a60a93569304a978c798bb8c4a4f7f52f", null ], + [ "flush_job_t", "intel-ipsec-mb_8h.html#ae64d0f677e42b8f8d50289113cf5e01e", null ], + [ "get_completed_job_t", "intel-ipsec-mb_8h.html#ac8cb9a545cdef14f60f36fe8888e6e7c", null ], + [ "get_next_job_t", "intel-ipsec-mb_8h.html#a5a533d852e3b65e9e5978fec37da43b5", null ], + [ "ghash_t", "intel-ipsec-mb_8h.html#a12e0c05f0ecec8d75d4b288a7ec11c84", null ], + [ "hash_fn_t", "intel-ipsec-mb_8h.html#afacacc8b74c05c80a0ee255c68637490", null ], + [ "hash_one_block_t", "intel-ipsec-mb_8h.html#aade58eb541a45650ed7d875dc2be97fa", null ], + [ "hec_32_t", "intel-ipsec-mb_8h.html#aa34b11bf4be97c764b283aedc0589575", null ], + [ "hec_64_t", "intel-ipsec-mb_8h.html#aa9afd4ee91f0d82729afcf5f1cdcb808", null ], + [ "IMB_JOB", "intel-ipsec-mb_8h.html#aed2387ba36d72c3d148fcb9d2802ccf5", null ], + [ "IMB_MGR", "intel-ipsec-mb_8h.html#a6d466496b4adfea3f9bc0881f11fe551", null ], + [ "init_mb_mgr_t", "intel-ipsec-mb_8h.html#af0c38ca0a5aac81d8db7e0cc2811426b", null ], + [ "kasumi_f8_1_buffer_bit_t", "intel-ipsec-mb_8h.html#a81afa744fb5669fe37cf0dce2193d8a7", null ], + [ "kasumi_f8_1_buffer_t", "intel-ipsec-mb_8h.html#a89fe975f0388b71765be494d04b034f4", null ], + [ "kasumi_f8_2_buffer_t", "intel-ipsec-mb_8h.html#af1eb074f5da469fa655d1404be8701dd", null ], + [ "kasumi_f8_3_buffer_t", "intel-ipsec-mb_8h.html#a53ecac7e009fcbeee603f975a45d3012", null ], + [ "kasumi_f8_4_buffer_t", "intel-ipsec-mb_8h.html#accf08a616e22a2c56e6a37d56f38b354", null ], + [ "kasumi_f8_n_buffer_t", "intel-ipsec-mb_8h.html#a51bbde186cdd437b6f9448e19a58e8a1", null ], + [ "kasumi_f9_1_buffer_t", "intel-ipsec-mb_8h.html#a8b02114f003b5fe0e1ccbfdc13ac55e4", null ], + [ "kasumi_f9_1_buffer_user_t", "intel-ipsec-mb_8h.html#ac17147d04602964bea59322ff31ebf59", null ], + [ "kasumi_init_f8_key_sched_t", "intel-ipsec-mb_8h.html#a7bec62666456d8962676961cf187ac80", null ], + [ "kasumi_init_f9_key_sched_t", "intel-ipsec-mb_8h.html#a8b30db8462aca1c039c3fd20646f342c", null ], + [ "kasumi_key_sched_size_t", "intel-ipsec-mb_8h.html#ad7444c36024a63025e7682c16946a00a", null ], + [ "kasumi_key_sched_t", "intel-ipsec-mb_8h.html#a82d6866ebcf1bd46b684fe473b4b73f2", null ], + [ "keyexp_t", "intel-ipsec-mb_8h.html#a60de89a391c8d6d9c51c742a87effb6f", null ], + [ "queue_size_t", "intel-ipsec-mb_8h.html#a9884ae92b598dd504f7d0ef8be5051d6", null ], + [ "snow3g_f8_1_buffer_bit_t", "intel-ipsec-mb_8h.html#a7bd00b31d12260305accf0d37b60c958", null ], + [ "snow3g_f8_1_buffer_t", "intel-ipsec-mb_8h.html#a281315fb8b441d837ebed541b3a23380", null ], + [ "snow3g_f8_2_buffer_t", "intel-ipsec-mb_8h.html#a4e8912e7e55a2ac891f85c19e7f29328", null ], + [ "snow3g_f8_4_buffer_t", "intel-ipsec-mb_8h.html#ae13b1154dcc13ca562e81471659d6099", null ], + [ "snow3g_f8_8_buffer_multikey_t", "intel-ipsec-mb_8h.html#a109b2a72d05bbe05a373ed5cf3c1f201", null ], + [ "snow3g_f8_8_buffer_t", "intel-ipsec-mb_8h.html#a6155f472c6794e88289235752d83154a", null ], + [ "snow3g_f8_n_buffer_multikey_t", "intel-ipsec-mb_8h.html#a4049b601f9da6542e25d903efe524bc1", null ], + [ "snow3g_f8_n_buffer_t", "intel-ipsec-mb_8h.html#a6a4a1da950207c9ed04f3f18ae1b7e43", null ], + [ "snow3g_f9_1_buffer_t", "intel-ipsec-mb_8h.html#ae7a48386f7190b4e46cd113eee413733", null ], + [ "snow3g_init_key_sched_t", "intel-ipsec-mb_8h.html#adfa182361f1c179d75129b4e46841af5", null ], + [ "snow3g_key_sched_size_t", "intel-ipsec-mb_8h.html#ad78a4f1c0d6de808f7046c0fbcd98d8f", null ], + [ "snow3g_key_schedule_t", "intel-ipsec-mb_8h.html#ae50b48832b41b4b468cf933b44849213", null ], + [ "submit_cipher_burst_t", "intel-ipsec-mb_8h.html#ac4bd89ab3cb1341eb4f30c4c501a9009", null ], + [ "submit_hash_burst_t", "intel-ipsec-mb_8h.html#a4f6fca648737cec9ee869786e9c240b5", null ], + [ "submit_job_t", "intel-ipsec-mb_8h.html#a41297d789b434157e746e338f0af96ec", null ], + [ "xcbc_keyexp_t", "intel-ipsec-mb_8h.html#abb36f71b9fb4d928c2ad9fa462b80ae6", null ], + [ "zuc_eea3_1_buffer_t", "intel-ipsec-mb_8h.html#a0a3032f0698c42bd486d053e90c44fdd", null ], + [ "zuc_eea3_4_buffer_t", "intel-ipsec-mb_8h.html#a7df2e3f063333d677ef7668626126c4d", null ], + [ "zuc_eea3_n_buffer_t", "intel-ipsec-mb_8h.html#a303f2dd11b1193938596b4ec69edf8cc", null ], + [ "zuc_eia3_1_buffer_t", "intel-ipsec-mb_8h.html#a95ed5580686f708ba2313179cd6064a8", null ], + [ "zuc_eia3_n_buffer_t", "intel-ipsec-mb_8h.html#ada45082bb04dd727c0deec5c4fc96fed", null ], + [ "IMB_ARCH", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6", [ + [ "IMB_ARCH_NONE", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1ff5121849f183accf83fd652b854733", null ], + [ "IMB_ARCH_NOAESNI", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a6a5b29422391adbb6806575bf7ccae44", null ], + [ "IMB_ARCH_SSE", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a3c2ccbe898a62cbaed5bad7717fdfa16", null ], + [ "IMB_ARCH_AVX", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6aa831f565e051aa13fcad5b224358e57d", null ], + [ "IMB_ARCH_AVX2", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a701628fddfdd84d56495839f20573de6", null ], + [ "IMB_ARCH_AVX512", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1df4f12c5769b52a95f1a4b89a8163c8", null ], + [ "IMB_ARCH_NUM", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6ae6536a65a8d0cd53faa325d5ee4dcdda", null ] + ] ], + [ "IMB_CHAIN_ORDER", "intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5", [ + [ "IMB_ORDER_CIPHER_HASH", "intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5e3ab845752ce3d16e40ea7802ce4a52", null ], + [ "IMB_ORDER_HASH_CIPHER", "intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5de8d032616267941d99dad14801044d", null ] + ] ], + [ "IMB_CIPHER_DIRECTION", "intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9ee", [ + [ "IMB_DIR_ENCRYPT", "intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eeabed12e936823e8f8b2de70388a2d15ce", null ], + [ "IMB_DIR_DECRYPT", "intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eea881652011955c227d94a881d2d2c5c59", null ] + ] ], + [ "IMB_CIPHER_MODE", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334c", [ + [ "IMB_CIPHER_CBC", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0e2ebfcfd101ab4a39129082b2de517f", null ], + [ "IMB_CIPHER_CNTR", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca5b4126e439eaff0975846425594201ea", null ], + [ "IMB_CIPHER_NULL", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0bebd24d7311dedce100455dd3a8ddf6", null ], + [ "IMB_CIPHER_DOCSIS_SEC_BPI", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caed8ba68d7d2d7602407f17305d1d99b7", null ], + [ "IMB_CIPHER_GCM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0f97344e525fef167a92d1a59a0492aa", null ], + [ "IMB_CIPHER_CUSTOM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca4301726c71a4faf412a1a011cc4f0df8", null ], + [ "IMB_CIPHER_DES", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca049e85047492494fac0a4ec785707851", null ], + [ "IMB_CIPHER_DOCSIS_DES", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caf9d62e1ff3a1a8b2ed1dcc515a3ffe1b", null ], + [ "IMB_CIPHER_CCM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca7d3946d77e4364bc45a9e72499bf817f", null ], + [ "IMB_CIPHER_DES3", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca639b1382de364a2b810f7876189bc64f", null ], + [ "IMB_CIPHER_PON_AES_CNTR", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca896285cd2fe56f73011b908e490cbbea", null ], + [ "IMB_CIPHER_ECB", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca357dec895f8dfa3371add1feaa001f56", null ], + [ "IMB_CIPHER_CNTR_BITLEN", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca6f69815480531651948b5549fa84023a", null ], + [ "IMB_CIPHER_ZUC_EEA3", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3618bfaab3c19aabc8a93a03ae121efc", null ], + [ "IMB_CIPHER_SNOW3G_UEA2_BITLEN", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca88c3ff9a6816b3629dc5b4815b47607e", null ], + [ "IMB_CIPHER_KASUMI_UEA1_BITLEN", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3f7a3edb272955efea015c5bb8572e3b", null ], + [ "IMB_CIPHER_CBCS_1_9", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca71d87659793f7377d1d8b3315a3559f8", null ], + [ "IMB_CIPHER_CHACHA20", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca39d8d7ebca424a1c1a65bc04b5f64741", null ], + [ "IMB_CIPHER_CHACHA20_POLY1305", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caa5c375f97dce274a2446c8abf90aa61a", null ], + [ "IMB_CIPHER_CHACHA20_POLY1305_SGL", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca40926d8b731a401ec3e86ffa2089a1eb", null ], + [ "IMB_CIPHER_SNOW_V", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cafea56410df85a0b006963e50fba78465", null ], + [ "IMB_CIPHER_SNOW_V_AEAD", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cada26994f8467300f862dbf45b5a22409", null ], + [ "IMB_CIPHER_GCM_SGL", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca48e024bc875c46b714b19a1c19908918", null ], + [ "IMB_CIPHER_NUM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca78c470d405fb36037e41224b99d23c75", null ] + ] ], + [ "IMB_ERR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0b", [ + [ "IMB_ERR_MIN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3f9d55f356a0caf413d2114a468ba659", null ], + [ "IMB_ERR_NULL_MBMGR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba9ba69cc32a2d17e3e7b4d4c982f26abd", null ], + [ "IMB_ERR_JOB_NULL_SRC", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4bc6a784afa9bb88c5afe4fd1b5f610d", null ], + [ "IMB_ERR_JOB_NULL_DST", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba55b3ddc1a48cb14ad020031b8d5b4927", null ], + [ "IMB_ERR_JOB_NULL_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba22402460340d6b34492ece390a7913c0", null ], + [ "IMB_ERR_JOB_NULL_IV", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0d35070d6476cde0202deb5531ae09c2", null ], + [ "IMB_ERR_JOB_NULL_AUTH", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba503544043c60c2195003c00b18141beb", null ], + [ "IMB_ERR_JOB_NULL_AAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3ef3d3180e3052862c796cf89320d60c", null ], + [ "IMB_ERR_JOB_CIPH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae1c9e58d4be3fdd7bb37247667472920", null ], + [ "IMB_ERR_JOB_AUTH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69dd17473dc4afb2f04c417fe6882608", null ], + [ "IMB_ERR_JOB_IV_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba390f91827f87bbd3233ad6e1f104b2b4", null ], + [ "IMB_ERR_JOB_KEY_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badfea107da94936d30900f23ad22b5285", null ], + [ "IMB_ERR_JOB_AUTH_TAG_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1b7e4c43808275e9cd63ca21091e5ab2", null ], + [ "IMB_ERR_JOB_AAD_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba31a6b672b95d7d100a97fa6d788c0dd6", null ], + [ "IMB_ERR_JOB_SRC_OFFSET", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba207979780b5d8472dfef2f5e12998d22", null ], + [ "IMB_ERR_JOB_CHAIN_ORDER", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacaec18a061b4bf792178408d69775950", null ], + [ "IMB_ERR_CIPH_MODE", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba61a411feda0e8cf5aff8c58ad2e92bdd", null ], + [ "IMB_ERR_HASH_ALGO", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8ac2802661bd9928d0c766129db2a814", null ], + [ "IMB_ERR_JOB_NULL_AUTH_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba97fb89f93299f67de86aac919bb83aa9", null ], + [ "IMB_ERR_JOB_NULL_SGL_CTX", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba05f9212af539a22ad65b12a9abc8e5c0", null ], + [ "IMB_ERR_JOB_NULL_NEXT_IV", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba983561202b1623b0bd1f691bd6036039", null ], + [ "IMB_ERR_JOB_PON_PLI", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bace7e834842ddef5b063c99544d00b303", null ], + [ "IMB_ERR_NULL_SRC", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba308819b7cbdadfd6bbee290408880302", null ], + [ "IMB_ERR_NULL_DST", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1a61dcc99a1a9725c89076dec3c02e02", null ], + [ "IMB_ERR_NULL_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba63029e3673eef74345ad80867f7ba0ef", null ], + [ "IMB_ERR_NULL_EXP_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3b60a570372529486da69c6d2b7be382", null ], + [ "IMB_ERR_NULL_IV", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafcfe882ee9730fd1724b587106f2f419", null ], + [ "IMB_ERR_NULL_AUTH", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1e20954ec8a501093175d640631550c1", null ], + [ "IMB_ERR_NULL_AAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badc7d17d90012d5c14fd8a12d9ea12543", null ], + [ "IMB_ERR_CIPH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bab5a2b4c1df7778c3efc236976062ed94", null ], + [ "IMB_ERR_AUTH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafe5af1c17e1caece14b3a5f659a1336d", null ], + [ "IMB_ERR_IV_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baac8500b58709f0bd05ad129337bb6af2", null ], + [ "IMB_ERR_KEY_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8d6b3ea721216bb12fbe40e72a5bd768", null ], + [ "IMB_ERR_AUTH_TAG_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6af5bd8fb7690b46afe3d2c1e623808e", null ], + [ "IMB_ERR_AAD_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0ad3ade3fe22c5887d9bf444ff3923aa", null ], + [ "IMB_ERR_SRC_OFFSET", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba20dfbdc8feb8332a9efe51de48fa6e09", null ], + [ "IMB_ERR_NULL_AUTH_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baa4f5aeab78ed40a4c19389b8f691306c", null ], + [ "IMB_ERR_NULL_CTX", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0b57ec134f6e10e11c3fbcf77ede282d", null ], + [ "IMB_ERR_NO_AESNI_EMU", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4496bf14e2ddf1133a384f9fbf96d5dc", null ], + [ "IMB_ERR_JOB_NULL_HMAC_OPAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacb9286af7533993ab79982b288aab97a", null ], + [ "IMB_ERR_JOB_NULL_HMAC_IPAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba15f55e87bb42cf5494dd01b9a1c4d712", null ], + [ "IMB_ERR_JOB_NULL_XCBC_K1_EXP", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69bf9e54fdf84b154284eab071c98804", null ], + [ "IMB_ERR_JOB_NULL_XCBC_K2", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bad7fbd09092821fb8605e11384dcdc428", null ], + [ "IMB_ERR_JOB_NULL_XCBC_K3", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baf12955208a99c8feeeb580fee940b7c3", null ], + [ "IMB_ERR_JOB_CIPH_DIR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacbf5d148be9963df1d46400ff26396db", null ], + [ "IMB_ERR_JOB_NULL_GHASH_INIT_TAG", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6c97704719f286b14204ca144bbdc7a9", null ], + [ "IMB_ERR_MISSING_CPUFLAGS_INIT_MGR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0babdd0b7bc92e7dcf166d36a9bcd118d60", null ], + [ "IMB_ERR_NULL_JOB", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba007c436f8e21f317e8f9225c5b009d38", null ], + [ "IMB_ERR_QUEUE_SPACE", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4f73ed8232981b84df1fee21024e01ac", null ], + [ "IMB_ERR_NULL_BURST", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba99d845a176cd571dd6f99b0a430291c6", null ], + [ "IMB_ERR_BURST_SIZE", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae2bc238b157942636836eaf0eedb5ddf", null ], + [ "IMB_ERR_BURST_OOO", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae4584575cf7134ba02ca1d38bb3f6444", null ], + [ "IMB_ERR_SELFTEST", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1873d38b8aaf5ebcd3f542b15c418829", null ], + [ "IMB_ERR_BURST_SUITE_ID", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba436eb6f9e045a02e3ad5998da8c13d78", null ], + [ "IMB_ERR_MAX", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bac773be615f9904fd69e961451fb0fac6", null ] + ] ], + [ "IMB_HASH_ALG", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006", [ + [ "IMB_AUTH_HMAC_SHA_1", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a441cc344587709b2ff2fc26f36ff1e58", null ], + [ "IMB_AUTH_HMAC_SHA_224", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad0ad42ef3cb8cef38a29bcc667e70fca", null ], + [ "IMB_AUTH_HMAC_SHA_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a60c1e71108516903aded4534f0f2e995", null ], + [ "IMB_AUTH_HMAC_SHA_384", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae98f8f8e42b7a026bb2aa2a402a291b9", null ], + [ "IMB_AUTH_HMAC_SHA_512", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad8f7bc68ad10fe75665692a27ada55f1", null ], + [ "IMB_AUTH_AES_XCBC", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a17deb61684701b7ec37f68bd34b6d454", null ], + [ "IMB_AUTH_MD5", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a958fb278e4745955ebc791ba652954fe", null ], + [ "IMB_AUTH_NULL", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a25f47297460775ab891a0abb8240b8e0", null ], + [ "IMB_AUTH_AES_GMAC", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa1496e45f00afe54aeeb527089e14a0d", null ], + [ "IMB_AUTH_CUSTOM", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a70151af8840242c8f5bdb375f2f1135b", null ], + [ "IMB_AUTH_AES_CCM", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a824187b143cb8d4b1778d10bcedbce3e", null ], + [ "IMB_AUTH_AES_CMAC", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aed85279dbff19be59c16216364375a48", null ], + [ "IMB_AUTH_SHA_1", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adcaf4a43c9c114bcdb75ff44ba1bea77", null ], + [ "IMB_AUTH_SHA_224", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3936068075b4572d859c9b1984312325", null ], + [ "IMB_AUTH_SHA_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4962dbdf6fcb8d743e4ab55ce23c4b88", null ], + [ "IMB_AUTH_SHA_384", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006afe74859581d19e70b8667773662081bc", null ], + [ "IMB_AUTH_SHA_512", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa12aca10078adbe0bc2a021190663e2d", null ], + [ "IMB_AUTH_AES_CMAC_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa9a94b55feed30ce57ecadf17b7c1a83", null ], + [ "IMB_AUTH_PON_CRC_BIP", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a2fcc080535b4dd16502ecd2b01c14048", null ], + [ "IMB_AUTH_ZUC_EIA3_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a188c2297dc5228820e690e8938cd407b", null ], + [ "IMB_AUTH_DOCSIS_CRC32", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae006c86697198ca852fc612d376135dc", null ], + [ "IMB_AUTH_SNOW3G_UIA2_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4eadf3270a81eb2567388843da2fa89b", null ], + [ "IMB_AUTH_KASUMI_UIA1", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a42dc1dc990059b61ef784813e0f8f49a", null ], + [ "IMB_AUTH_AES_GMAC_128", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a189af960a1558b7ac3289c0892739ea0", null ], + [ "IMB_AUTH_AES_GMAC_192", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adfd8db9df85ef929d9e4a38612f07d66", null ], + [ "IMB_AUTH_AES_GMAC_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae6fab01dbaf8b0110ca8ccdce98547af", null ], + [ "IMB_AUTH_AES_CMAC_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a0d0cad9513b19110454c9e4e9c549205", null ], + [ "IMB_AUTH_POLY1305", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a93e6c082f7ec5926a087d139a2bd2e56", null ], + [ "IMB_AUTH_CHACHA20_POLY1305", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab737fa5ff7244ddf5024905f6477ae01", null ], + [ "IMB_AUTH_CHACHA20_POLY1305_SGL", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a71119edfbe668d423e910b11c0efb712", null ], + [ "IMB_AUTH_ZUC256_EIA3_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a7c535075abfb0878168a00600a762fe0", null ], + [ "IMB_AUTH_SNOW_V_AEAD", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff331c62b01fb8427e63e18fc433f6", null ], + [ "IMB_AUTH_GCM_SGL", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ada7d3fc165b6855ae3e83467de9916b2", null ], + [ "IMB_AUTH_CRC32_ETHERNET_FCS", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a421294f9a35f87e796f96b48257e6dc2", null ], + [ "IMB_AUTH_CRC32_SCTP", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff8b335b5391fc33c80c076590de44", null ], + [ "IMB_AUTH_CRC32_WIMAX_OFDMA_DATA", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad3936131d6804e6c6aa3bd497ae675ab", null ], + [ "IMB_AUTH_CRC24_LTE_A", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3a8aa07fe9f70ece22f9351db0f8bb31", null ], + [ "IMB_AUTH_CRC24_LTE_B", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a77ccdf9abd97310f6808c8cf12c1a395", null ], + [ "IMB_AUTH_CRC16_X25", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a22e2e766cbec475508af01c6a722c2c3", null ], + [ "IMB_AUTH_CRC16_FP_DATA", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa78e695260f8fe6bb56d242771624ccc", null ], + [ "IMB_AUTH_CRC11_FP_HEADER", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab0f02ac14a03f137a83bb7d6dff539fc", null ], + [ "IMB_AUTH_CRC10_IUUP_DATA", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ac27de87b2687b1e2a293544be6d3fab9", null ], + [ "IMB_AUTH_CRC8_WIMAX_OFDMA_HCS", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae62fe411b683840a5f25c199b66c7f42", null ], + [ "IMB_AUTH_CRC7_FP_HEADER", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab7f1372ade9da040caf195b13f2b79f0", null ], + [ "IMB_AUTH_CRC6_IUUP_HEADER", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a88175070e077037332ad5cd5fdece46f", null ], + [ "IMB_AUTH_GHASH", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a67297eacd4913bd65a1aae650c3a22e3", null ], + [ "IMB_AUTH_NUM", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a5d34120f8a1c5cde857baf32536868af", null ] + ] ], + [ "IMB_KEY_SIZE_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cc", [ + [ "IMB_KEY_64_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaa1eb9eb83fca5f5dd6578f27733eb390", null ], + [ "IMB_KEY_128_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaf3e82db9a8b5b1b229c201735938969d", null ], + [ "IMB_KEY_192_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cca573d62aa6a04d9ce2b3c34e226846598", null ], + [ "IMB_KEY_256_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccac139e7555c9b30725359698833eb213a", null ] + ] ], + [ "IMB_SGL_STATE", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3d", [ + [ "IMB_SGL_INIT", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3daf6e77a48759ab366cc1160725ff4261e", null ], + [ "IMB_SGL_UPDATE", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da15c5d0aeeb9d5eb4e4c4878c837c5977", null ], + [ "IMB_SGL_COMPLETE", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3dab7a602903c0fbaad7ffb893cb2547975", null ], + [ "IMB_SGL_ALL", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da2f0678ab7ad9bb30c040e0b3ee2fa408", null ] + ] ], + [ "IMB_STATUS", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066ab", [ + [ "IMB_STATUS_BEING_PROCESSED", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaa562b01ade81e0196fc23cd79d534141", null ], + [ "IMB_STATUS_COMPLETED_CIPHER", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba6fcaccd3853c9e8cf3b402edc3e22089", null ], + [ "IMB_STATUS_COMPLETED_AUTH", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba79df83a15cdafc2a6c79751457ae9792", null ], + [ "IMB_STATUS_COMPLETED", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba508a9f195acee09621542ca687ba2341", null ], + [ "IMB_STATUS_INVALID_ARGS", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaee85b6a8bc33e0b39001e9ff538540b7", null ], + [ "IMB_STATUS_INTERNAL_ERROR", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba546465b9d4cfb1f9c98c864bf5ed51fb", null ], + [ "IMB_STATUS_ERROR", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba0ef40895610ac6343d171f65be676a1d", null ] + ] ], + [ "aes_cmac_subkey_gen_avx", "intel-ipsec-mb_8h.html#a6541825066a8d2b8b3c7b6e2c4b28c83", null ], + [ "aes_cmac_subkey_gen_avx2", "intel-ipsec-mb_8h.html#a6903e1e0dfb8fd75c2f556596f3efa1b", null ], + [ "aes_cmac_subkey_gen_avx512", "intel-ipsec-mb_8h.html#a730f18bd7cdcbbb995e24931307867fc", null ], + [ "aes_cmac_subkey_gen_sse", "intel-ipsec-mb_8h.html#a4dafe82f4933670f68e1bf34d55a691d", null ], + [ "aes_gcm_dec_128_avx_gen2", "intel-ipsec-mb_8h.html#a8c9a7e51599dbce89a416e0d8422742b", null ], + [ "aes_gcm_dec_128_avx_gen4", "intel-ipsec-mb_8h.html#ae1e8e1b4425af520048a9e1fe5689e4d", null ], + [ "aes_gcm_dec_128_finalize_avx_gen2", "intel-ipsec-mb_8h.html#ac173e4fb252a820edb25ad790065ad39", null ], + [ "aes_gcm_dec_128_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a8e2bc4fff9de3c70ac140ff30e1cdce0", null ], + [ "aes_gcm_dec_128_finalize_sse", "intel-ipsec-mb_8h.html#a08603861ac18d1dce7b322b0c3fb69c5", null ], + [ "aes_gcm_dec_128_sse", "intel-ipsec-mb_8h.html#add2f5aafaf55cd0e0b20b2b62fcb025d", null ], + [ "aes_gcm_dec_128_update_avx_gen2", "intel-ipsec-mb_8h.html#a796e1df6d2aa233a6e28a867c1a2c628", null ], + [ "aes_gcm_dec_128_update_avx_gen4", "intel-ipsec-mb_8h.html#a4464684ff4cf4e9cb978409ad350c4d6", null ], + [ "aes_gcm_dec_128_update_sse", "intel-ipsec-mb_8h.html#a2216e461e916668b70d547513b0cf26f", null ], + [ "aes_gcm_dec_192_avx_gen2", "intel-ipsec-mb_8h.html#ad45e83e263506684c786b4e7012055ac", null ], + [ "aes_gcm_dec_192_avx_gen4", "intel-ipsec-mb_8h.html#a5ccd87fbcfffc238fbcd1fc816eb06ee", null ], + [ "aes_gcm_dec_192_finalize_avx_gen2", "intel-ipsec-mb_8h.html#aacc19f04c9ac80d1bf7391661e191798", null ], + [ "aes_gcm_dec_192_finalize_avx_gen4", "intel-ipsec-mb_8h.html#afa136a3c06d45e914d15415dee27f001", null ], + [ "aes_gcm_dec_192_finalize_sse", "intel-ipsec-mb_8h.html#a0fd357a1794d78ac2168bc7eff0442ce", null ], + [ "aes_gcm_dec_192_sse", "intel-ipsec-mb_8h.html#acaff83c8ca568201805e0ada5a79f6e1", null ], + [ "aes_gcm_dec_192_update_avx_gen2", "intel-ipsec-mb_8h.html#aa81615f7ddfc8e1a794c5e278d22f7fc", null ], + [ "aes_gcm_dec_192_update_avx_gen4", "intel-ipsec-mb_8h.html#a615c26f03ba08c8cde73cb074bf6b363", null ], + [ "aes_gcm_dec_192_update_sse", "intel-ipsec-mb_8h.html#a58455aec5650ecfc6077c69c97031f63", null ], + [ "aes_gcm_dec_256_avx_gen2", "intel-ipsec-mb_8h.html#a9b79136dc5996baf598e3aeee520871c", null ], + [ "aes_gcm_dec_256_avx_gen4", "intel-ipsec-mb_8h.html#ac16384799933d4f7b11cb56d9c66f5da", null ], + [ "aes_gcm_dec_256_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a72f76caa84e2803b2b4a26c81623f89e", null ], + [ "aes_gcm_dec_256_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a791be274839adc3d26d9a32aa319dc3b", null ], + [ "aes_gcm_dec_256_finalize_sse", "intel-ipsec-mb_8h.html#adf7aa2e79dce7a9ed79134c086715fd0", null ], + [ "aes_gcm_dec_256_sse", "intel-ipsec-mb_8h.html#afd8763c66749e9bc5e4e5eb2ac130e43", null ], + [ "aes_gcm_dec_256_update_avx_gen2", "intel-ipsec-mb_8h.html#a0c31c2e2760edc7b252214214159cd7a", null ], + [ "aes_gcm_dec_256_update_avx_gen4", "intel-ipsec-mb_8h.html#a141303c38f4eed1245b542b6e9473296", null ], + [ "aes_gcm_dec_256_update_sse", "intel-ipsec-mb_8h.html#ab1a51ae4b3cf3e8fdb618773d7b24b68", null ], + [ "aes_gcm_enc_128_avx_gen2", "intel-ipsec-mb_8h.html#af33d7afa861dcedc2c284330a5a66164", null ], + [ "aes_gcm_enc_128_avx_gen4", "intel-ipsec-mb_8h.html#aa17c919e873b6cfd7ac990977eef7601", null ], + [ "aes_gcm_enc_128_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a40a83ee6dec0440cd99e5ff600e3803d", null ], + [ "aes_gcm_enc_128_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a004c38b922d25d8631252854426ebf44", null ], + [ "aes_gcm_enc_128_finalize_sse", "intel-ipsec-mb_8h.html#af63ae2246c6d89b7d6630623948d3886", null ], + [ "aes_gcm_enc_128_sse", "intel-ipsec-mb_8h.html#aad9e6c466a5274b974d20330f91add50", null ], + [ "aes_gcm_enc_128_update_avx_gen2", "intel-ipsec-mb_8h.html#ae3459d38a2ee2e1ce04b9f9f4bf55ea5", null ], + [ "aes_gcm_enc_128_update_avx_gen4", "intel-ipsec-mb_8h.html#abb5570a161a90391e25b2016bc98381b", null ], + [ "aes_gcm_enc_128_update_sse", "intel-ipsec-mb_8h.html#a7b12cabbfebb420022a19124aff78a3f", null ], + [ "aes_gcm_enc_192_avx_gen2", "intel-ipsec-mb_8h.html#a0ff9569b5072896a4119a4c7d3849bfc", null ], + [ "aes_gcm_enc_192_avx_gen4", "intel-ipsec-mb_8h.html#a9b462a3c00471e4338f0bae7d5abbda7", null ], + [ "aes_gcm_enc_192_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a5177757ae4a75ba41d5e7a8a534edc06", null ], + [ "aes_gcm_enc_192_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a4f76beaee47ffeeeb6f94a132f898339", null ], + [ "aes_gcm_enc_192_finalize_sse", "intel-ipsec-mb_8h.html#a5dcb5c37d1a5377967888e9b76208f8c", null ], + [ "aes_gcm_enc_192_sse", "intel-ipsec-mb_8h.html#adec2f9778622b3e226f12bb326e75076", null ], + [ "aes_gcm_enc_192_update_avx_gen2", "intel-ipsec-mb_8h.html#a548fcbb89ff5fb798bf71ab8e4ac88eb", null ], + [ "aes_gcm_enc_192_update_avx_gen4", "intel-ipsec-mb_8h.html#a174447b05661285b2729805f66bcef81", null ], + [ "aes_gcm_enc_192_update_sse", "intel-ipsec-mb_8h.html#a46c3df131f9e1e3ed1a283349c4072f7", null ], + [ "aes_gcm_enc_256_avx_gen2", "intel-ipsec-mb_8h.html#a45dea093699e6e2dd462b6cafccee672", null ], + [ "aes_gcm_enc_256_avx_gen4", "intel-ipsec-mb_8h.html#a52c0913f10dd41fea5fe328037561ff8", null ], + [ "aes_gcm_enc_256_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a7fa5d4f44bb44600fa7b86697ccc385f", null ], + [ "aes_gcm_enc_256_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a873a75e2175acd3741b5cfc41180fe2c", null ], + [ "aes_gcm_enc_256_finalize_sse", "intel-ipsec-mb_8h.html#afcdedeabc7516e7506fa5c81d07f6fd8", null ], + [ "aes_gcm_enc_256_sse", "intel-ipsec-mb_8h.html#a460ad1ebcb7df39e39c9aca214d8109c", null ], + [ "aes_gcm_enc_256_update_avx_gen2", "intel-ipsec-mb_8h.html#a351c2bf07dcab7e01570273c9df1e56f", null ], + [ "aes_gcm_enc_256_update_avx_gen4", "intel-ipsec-mb_8h.html#ad9592051cf26814ea5830826b553c547", null ], + [ "aes_gcm_enc_256_update_sse", "intel-ipsec-mb_8h.html#ab831bdb2193a5adc54b51285f23ad5ce", null ], + [ "aes_gcm_init_128_avx_gen2", "intel-ipsec-mb_8h.html#a7a9a24d480dcedac1aa40d53e01b8a76", null ], + [ "aes_gcm_init_128_avx_gen4", "intel-ipsec-mb_8h.html#a8d225d08a5cf144208dacb155e192cb6", null ], + [ "aes_gcm_init_128_sse", "intel-ipsec-mb_8h.html#aca7ab49c370f8f7cb82f93db199bfd1c", null ], + [ "aes_gcm_init_192_avx_gen2", "intel-ipsec-mb_8h.html#ac24548c2955399fd84b03216ad5b36fb", null ], + [ "aes_gcm_init_192_avx_gen4", "intel-ipsec-mb_8h.html#a757cd199a2e993c6610bbcdfc1da1ae4", null ], + [ "aes_gcm_init_192_sse", "intel-ipsec-mb_8h.html#a5bbf84135701f572ec2ecb0c0898848b", null ], + [ "aes_gcm_init_256_avx_gen2", "intel-ipsec-mb_8h.html#a3dd74260f4f34f7a53253c63ad3a6701", null ], + [ "aes_gcm_init_256_avx_gen4", "intel-ipsec-mb_8h.html#a99310b73bc9cde6b69b682e72cd0efc1", null ], + [ "aes_gcm_init_256_sse", "intel-ipsec-mb_8h.html#a09949bd5112eec14773dce239ca39033", null ], + [ "aes_gcm_pre_128_avx_gen2", "intel-ipsec-mb_8h.html#a2f60098fff25e8b71b35e42cc7854339", null ], + [ "aes_gcm_pre_128_avx_gen4", "intel-ipsec-mb_8h.html#a3d7b8bc454b44abbeaa8f435280f8cb1", null ], + [ "aes_gcm_pre_128_sse", "intel-ipsec-mb_8h.html#a5d02ac2586befa6cd3fd4921858c33fd", null ], + [ "aes_gcm_pre_192_avx_gen2", "intel-ipsec-mb_8h.html#a6c9c0b179633aca3ba5c520bee32662f", null ], + [ "aes_gcm_pre_192_avx_gen4", "intel-ipsec-mb_8h.html#ab71c95df9bd059e1a5e108d2513a0e2c", null ], + [ "aes_gcm_pre_192_sse", "intel-ipsec-mb_8h.html#ac0ebc1e7678bde75c25d5c0ec7ab2f7a", null ], + [ "aes_gcm_pre_256_avx_gen2", "intel-ipsec-mb_8h.html#aaaabf4acdd43f98b6c2d31c8013ebbf4", null ], + [ "aes_gcm_pre_256_avx_gen4", "intel-ipsec-mb_8h.html#a082773f106f399ca73afc56583805a8c", null ], + [ "aes_gcm_pre_256_sse", "intel-ipsec-mb_8h.html#a18fac8ea0f3acabb4d5ffab1152f477f", null ], + [ "aes_gcm_precomp_128_avx_gen2", "intel-ipsec-mb_8h.html#ade16ed4c31baa50f8a10e3c85156a942", null ], + [ "aes_gcm_precomp_128_avx_gen4", "intel-ipsec-mb_8h.html#aecf7277999128599c6cf8432b242ba36", null ], + [ "aes_gcm_precomp_128_sse", "intel-ipsec-mb_8h.html#a46253857355ddae8d17bab1403289d0f", null ], + [ "aes_gcm_precomp_192_avx_gen2", "intel-ipsec-mb_8h.html#ab0055e3b7d00cdb578647921207a1de7", null ], + [ "aes_gcm_precomp_192_avx_gen4", "intel-ipsec-mb_8h.html#a6cb54cf9321dfb95af2eeb812f41fa8f", null ], + [ "aes_gcm_precomp_192_sse", "intel-ipsec-mb_8h.html#a72d93e3249806eee186d465dc04c09e6", null ], + [ "aes_gcm_precomp_256_avx_gen2", "intel-ipsec-mb_8h.html#a2a883085207ee21caca109ab8250db08", null ], + [ "aes_gcm_precomp_256_avx_gen4", "intel-ipsec-mb_8h.html#abef922c52013630ad70273cef095c564", null ], + [ "aes_gcm_precomp_256_sse", "intel-ipsec-mb_8h.html#a18001897ea80cdce1f4f608b0922f767", null ], + [ "aes_keyexp_128_avx", "intel-ipsec-mb_8h.html#aec7fe23c32ee94de88e5feba8f373a04", null ], + [ "aes_keyexp_128_avx2", "intel-ipsec-mb_8h.html#a4fc7f88bdb263ae0d45f37af7e4cb930", null ], + [ "aes_keyexp_128_avx512", "intel-ipsec-mb_8h.html#a9b7faaf3e5235d0a3c4b784f1035896d", null ], + [ "aes_keyexp_128_enc_avx", "intel-ipsec-mb_8h.html#ac7d7c889a25730d82ddee27bad7f0a63", null ], + [ "aes_keyexp_128_enc_avx2", "intel-ipsec-mb_8h.html#a84e76bdfb5a69dba7b515dd0b4fb4312", null ], + [ "aes_keyexp_128_enc_avx512", "intel-ipsec-mb_8h.html#a68e6a0e99a2091d3630bf3315fe9d95c", null ], + [ "aes_keyexp_128_enc_sse", "intel-ipsec-mb_8h.html#a270621a2f73134d8d021b5dd57ad6817", null ], + [ "aes_keyexp_128_sse", "intel-ipsec-mb_8h.html#a32bcda3484b33b7d01124dbfeb9a229b", null ], + [ "aes_keyexp_192_avx", "intel-ipsec-mb_8h.html#a3bc56101b48217094086ae3120b0e086", null ], + [ "aes_keyexp_192_avx2", "intel-ipsec-mb_8h.html#aba6dd2e633819829740cb7917ba6b1b2", null ], + [ "aes_keyexp_192_avx512", "intel-ipsec-mb_8h.html#a1f69c0a43de83f8fba19107a63c07831", null ], + [ "aes_keyexp_192_enc_avx", "intel-ipsec-mb_8h.html#a5350b81ab574c990d6a5b9dc3b203c51", null ], + [ "aes_keyexp_192_enc_avx2", "intel-ipsec-mb_8h.html#a460853c072adc3a0f431e861d3e5bba1", null ], + [ "aes_keyexp_192_enc_avx512", "intel-ipsec-mb_8h.html#a72807e8300208b22b5d532e274385127", null ], + [ "aes_keyexp_192_enc_sse", "intel-ipsec-mb_8h.html#a0a843e722f2af7a645340f6a98cfa4c2", null ], + [ "aes_keyexp_192_sse", "intel-ipsec-mb_8h.html#a7681d777bb5f991ab59b5581bc854562", null ], + [ "aes_keyexp_256_avx", "intel-ipsec-mb_8h.html#a5b92ee3873b2e2cd02c9312b6040ee91", null ], + [ "aes_keyexp_256_avx2", "intel-ipsec-mb_8h.html#af1f440855a00edbf95c124a1e1d18c37", null ], + [ "aes_keyexp_256_avx512", "intel-ipsec-mb_8h.html#a37e0f61656618413fcfd919dfe79ca45", null ], + [ "aes_keyexp_256_enc_avx", "intel-ipsec-mb_8h.html#a22096fdd9b5d32095571f04a838d66a9", null ], + [ "aes_keyexp_256_enc_avx2", "intel-ipsec-mb_8h.html#a73da5625112ad845a3ec94ee84590f54", null ], + [ "aes_keyexp_256_enc_avx512", "intel-ipsec-mb_8h.html#a9fb47323b270bffc87ad8e77985e827d", null ], + [ "aes_keyexp_256_enc_sse", "intel-ipsec-mb_8h.html#afa369dee8def2ef620e7f33db86a3356", null ], + [ "aes_keyexp_256_sse", "intel-ipsec-mb_8h.html#a5105a8d995cd8e4354c7de7ac94886cb", null ], + [ "aes_xcbc_expand_key_avx", "intel-ipsec-mb_8h.html#afe460e4ff6a7b0c7a6edcc87e9976121", null ], + [ "aes_xcbc_expand_key_avx2", "intel-ipsec-mb_8h.html#ac3f27672ce30d74f5cdc28f1903f68b1", null ], + [ "aes_xcbc_expand_key_avx512", "intel-ipsec-mb_8h.html#a1308be648182642b8d73ff27d9344564", null ], + [ "aes_xcbc_expand_key_sse", "intel-ipsec-mb_8h.html#aefb28e46620cbf65a2af3073b2401e40", null ], + [ "alloc_mb_mgr", "intel-ipsec-mb_8h.html#ab8dc617feb010b0f4cd74fd9e1eabb5e", null ], + [ "des_cfb_one", "intel-ipsec-mb_8h.html#a71a0353411f5307a2177ae9b2814f522", null ], + [ "des_key_schedule", "intel-ipsec-mb_8h.html#a9911462fe4689c7420b827bb13fc3440", null ], + [ "flush_job_avx", "intel-ipsec-mb_8h.html#abc6b0f1c857f26b834f22b6bd8f51846", null ], + [ "flush_job_avx2", "intel-ipsec-mb_8h.html#ae242d142065172898129f30e6bc40d8c", null ], + [ "flush_job_avx512", "intel-ipsec-mb_8h.html#ac7aec54f653aeee2554ba980d62fd491", null ], + [ "flush_job_sse", "intel-ipsec-mb_8h.html#a96b880b8a32b0466f880d4f22d95cc3b", null ], + [ "free_mb_mgr", "intel-ipsec-mb_8h.html#a238b96cc36c0dc64798a1244b94add8b", null ], + [ "get_completed_job_avx", "intel-ipsec-mb_8h.html#a9d115c47670330bd01b960f2188169d9", null ], + [ "get_completed_job_avx2", "intel-ipsec-mb_8h.html#ad85bdbf56038fd72ceda7e50deea586e", null ], + [ "get_completed_job_avx512", "intel-ipsec-mb_8h.html#a693afd528fcc0809cc667b6c38de84dc", null ], + [ "get_completed_job_sse", "intel-ipsec-mb_8h.html#a267a718472e0c1b6585402e81e44e7e9", null ], + [ "get_next_job_avx", "intel-ipsec-mb_8h.html#a3518b02f06a82e9f83cd0435f3cf25c9", null ], + [ "get_next_job_avx2", "intel-ipsec-mb_8h.html#a49a3d54979188b3c68bdd621043125f7", null ], + [ "get_next_job_avx512", "intel-ipsec-mb_8h.html#a40c08ba2c387e24f2096b1213c46c812", null ], + [ "get_next_job_sse", "intel-ipsec-mb_8h.html#a055db1f6f16bc17724bdc64d0f313871", null ], + [ "imb_clear_mem", "intel-ipsec-mb_8h.html#aecdd69541b6d2a83d19ca804ef4b78d6", null ], + [ "imb_get_errno", "intel-ipsec-mb_8h.html#a79f60b2bffbe91dac3e4f8b486e7ab10", null ], + [ "imb_get_feature_flags", "intel-ipsec-mb_8h.html#a725703ff6da6b402bde7041a2fabfddf", null ], + [ "imb_get_mb_mgr_size", "intel-ipsec-mb_8h.html#ac8379ada0d6ec030ebad9828fbea9459", null ], + [ "imb_get_strerror", "intel-ipsec-mb_8h.html#a61f664448a862fbb02e728ce518ff8e4", null ], + [ "imb_get_version", "intel-ipsec-mb_8h.html#aee02bcb3a5e5eaee3632086f75c3d78d", null ], + [ "imb_get_version_str", "intel-ipsec-mb_8h.html#a4490e7b324236315f1a9e141b39f99f2", null ], + [ "imb_hmac_ipad_opad", "intel-ipsec-mb_8h.html#ab5ecb9fd270d0bfc42e46c5a3a9225fe", null ], + [ "imb_quic_aes_gcm", "intel-ipsec-mb_8h.html#a8faae8f9387983904125d862c4cac9ab", null ], + [ "imb_quic_hp_aes_ecb", "intel-ipsec-mb_8h.html#a89f9807d10f3459444bb649caf9b64ca", null ], + [ "imb_set_pointers_mb_mgr", "intel-ipsec-mb_8h.html#a3a825e903d1f8f8611b98b60fcaa5a81", null ], + [ "imb_set_session", "intel-ipsec-mb_8h.html#ada304337e78ef5a462c97c1f8151ef67", null ], + [ "init_mb_mgr_auto", "intel-ipsec-mb_8h.html#a2cbbd032835837fd0aa83b04f5337638", null ], + [ "init_mb_mgr_avx", "intel-ipsec-mb_8h.html#aee547b1b9e70e12c84d9949317e41c6c", null ], + [ "init_mb_mgr_avx2", "intel-ipsec-mb_8h.html#a71b6563b6b4c0c504cd9b8302a242cfa", null ], + [ "init_mb_mgr_avx512", "intel-ipsec-mb_8h.html#a98932a3c61d798b0c8a560dcc3dce16f", null ], + [ "init_mb_mgr_sse", "intel-ipsec-mb_8h.html#a2ad1941fe5d77b8bb2d6445679e3b0c5", null ], + [ "kasumi_f8_iv_gen", "intel-ipsec-mb_8h.html#a3876dca52c5c210dd0827b2a472e6d0b", null ], + [ "kasumi_f9_iv_gen", "intel-ipsec-mb_8h.html#ac886170fc4e70b467383fdf09f0724cf", null ], + [ "md5_one_block_avx", "intel-ipsec-mb_8h.html#a7addeb4d0c6ddcdaa031b9e7b4dd35c2", null ], + [ "md5_one_block_avx2", "intel-ipsec-mb_8h.html#a8b912812c65f4d3b592875cdd233704d", null ], + [ "md5_one_block_avx512", "intel-ipsec-mb_8h.html#ac7f4da2fc0601ad50851e7e43f53e0cf", null ], + [ "md5_one_block_sse", "intel-ipsec-mb_8h.html#a9d883a3a629240fe86efedd180294433", null ], + [ "queue_size_avx", "intel-ipsec-mb_8h.html#a800c395d326e718a2b520d0bb91cbb89", null ], + [ "queue_size_avx2", "intel-ipsec-mb_8h.html#a8a540000c08366ad47d95e23ed5b2918", null ], + [ "queue_size_avx512", "intel-ipsec-mb_8h.html#a51c0426a4b57c9e7db2b091692dff472", null ], + [ "queue_size_sse", "intel-ipsec-mb_8h.html#a80f792744558de173ea97d722c58e5c1", null ], + [ "snow3g_f8_iv_gen", "intel-ipsec-mb_8h.html#a2cbdb64c0b305cb5af010f9c98f70a43", null ], + [ "snow3g_f9_iv_gen", "intel-ipsec-mb_8h.html#a93e7a99927bc37fb4cb03f966fa4ebdd", null ], + [ "submit_job_avx", "intel-ipsec-mb_8h.html#a950bf3b7a3c63a371b561ea20aaf3a18", null ], + [ "submit_job_avx2", "intel-ipsec-mb_8h.html#a8913995859e45005f8de57a3456c93cf", null ], + [ "submit_job_avx512", "intel-ipsec-mb_8h.html#a75cc0d090470e47cd8be04451ee31a68", null ], + [ "submit_job_nocheck_avx", "intel-ipsec-mb_8h.html#ac7b339426753d3a7c44625a5ded3170a", null ], + [ "submit_job_nocheck_avx2", "intel-ipsec-mb_8h.html#a72caaf8162ae6f802f7c1fe570afe42e", null ], + [ "submit_job_nocheck_avx512", "intel-ipsec-mb_8h.html#a5a95b9268508d68b9027df98167d22cd", null ], + [ "submit_job_nocheck_sse", "intel-ipsec-mb_8h.html#a35e4c8a3d2eba8a612f021572621f98e", null ], + [ "submit_job_sse", "intel-ipsec-mb_8h.html#a7700e90058a74980bfd7d462a19d32a0", null ], + [ "zuc_eea3_iv_gen", "intel-ipsec-mb_8h.html#a4931dd6b1b91456bfe885e28ada0aaf3", null ], + [ "zuc_eia3_iv_gen", "intel-ipsec-mb_8h.html#ae3cefd0e992f1390110f0ef9a3aa510f", null ] +]; \ No newline at end of file diff --git a/docs/intel-ipsec-mb_8h_source.html b/docs/intel-ipsec-mb_8h_source.html new file mode 100644 index 0000000000000000000000000000000000000000..c0d663f8a15c2f8e7dfa07361bd70dae4ce2267b --- /dev/null +++ b/docs/intel-ipsec-mb_8h_source.html @@ -0,0 +1,2752 @@ + + + + + + + +intel-ipsec-mb: intel-ipsec-mb.h Source File + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
intel-ipsec-mb.h
+
+
+Go to the documentation of this file.
1/*******************************************************************************
+
2 Copyright (c) 2012-2023, Intel Corporation
+
3
+
4 Redistribution and use in source and binary forms, with or without
+
5 modification, are permitted provided that the following conditions are met:
+
6
+
7 * Redistributions of source code must retain the above copyright notice,
+
8 this list of conditions and the following disclaimer.
+
9 * Redistributions in binary form must reproduce the above copyright
+
10 notice, this list of conditions and the following disclaimer in the
+
11 documentation and/or other materials provided with the distribution.
+
12 * Neither the name of Intel Corporation nor the names of its contributors
+
13 may be used to endorse or promote products derived from this software
+
14 without specific prior written permission.
+
15
+
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+
20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+
21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+
22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+
23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+
24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+
25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
26*******************************************************************************/
+
27
+
28#ifndef IMB_IPSEC_MB_H
+
29#define IMB_IPSEC_MB_H
+
30
+
31#include <stdlib.h>
+
32#include <stdint.h>
+
33#include <errno.h>
+
34
+
35#ifdef __cplusplus
+
36extern "C" {
+
37#endif
+
38
+
39/* 128-bit data type that is not in sdtint.h */
+
40typedef struct {
+
41 uint64_t low;
+
42 uint64_t high;
+
43} imb_uint128_t;
+
44
+
48#if defined __linux__ || defined __FreeBSD__
+
50#define DECLARE_ALIGNED(decl, alignval) \
+
51 decl __attribute__((aligned(alignval)))
+
52#define __forceinline \
+
53 static inline __attribute__((always_inline))
+
54
+
55#if __GNUC__ >= 4
+
56#define IMB_DLL_EXPORT __attribute__((visibility("default")))
+
57#define IMB_DLL_LOCAL __attribute__((visibility("hidden")))
+
58#else /* GNU C 4.0 and later */
+
59#define IMB_DLL_EXPORT
+
60#define IMB_DLL_LOCAL
+
61#endif
+
63#else
+
64/* Windows */
+
65
+
66#ifdef __MINGW32__
+
67/* MinGW-w64 */
+
68#define DECLARE_ALIGNED(decl, alignval) \
+
69 decl __attribute__((aligned(alignval)))
+
70#undef __forceinline
+
71#define __forceinline \
+
72 static inline __attribute__((always_inline))
+
73
+
74#else
+
75/* MSVS */
+
76#define DECLARE_ALIGNED(decl, alignval) \
+
77 __declspec(align(alignval)) decl
+
78#define __forceinline \
+
79 static __forceinline
+
80
+
81#endif /* __MINGW__ */
+
82
+
86#define IMB_DLL_EXPORT
+
87#define IMB_DLL_LOCAL
+
88
+
89#endif /* defined __linux__ || defined __FreeBSD__ */
+
90
+
94#define IMB_VERSION_STR "1.4.0-dev"
+
95#define IMB_VERSION_NUM 0x10400
+
96
+
100#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+
101
+
105#ifdef DEBUG
+
106#include <assert.h>
+
107#define IMB_ASSERT(x) assert(x)
+
108#else
+
109#define IMB_ASSERT(x)
+
110#endif
+
111
+
112#ifndef IMB_DIM
+
113#define IMB_DIM(x) (sizeof(x) / sizeof(x[0]))
+
114#endif
+
115
+
119typedef enum {
+
120 IMB_ARCH_NONE = 0,
+
121 IMB_ARCH_NOAESNI,
+
122 IMB_ARCH_SSE,
+
123 IMB_ARCH_AVX,
+
124 IMB_ARCH_AVX2,
+
125 IMB_ARCH_AVX512,
+
126 IMB_ARCH_NUM,
+
127} IMB_ARCH;
+
128
+
132#define IMB_DES_KEY_SCHED_SIZE (16 * 8)
+
133#define IMB_DES_BLOCK_SIZE 8
+
134
+
135#define IMB_AES_BLOCK_SIZE 16
+
136
+
137#define IMB_SHA1_DIGEST_SIZE_IN_BYTES 20
+
138#define IMB_SHA224_DIGEST_SIZE_IN_BYTES 28
+
139#define IMB_SHA256_DIGEST_SIZE_IN_BYTES 32
+
140#define IMB_SHA384_DIGEST_SIZE_IN_BYTES 48
+
141#define IMB_SHA512_DIGEST_SIZE_IN_BYTES 64
+
142
+
143#define IMB_MD5_DIGEST_SIZE_IN_BYTES 16
+
144
+
145#define IMB_SHA1_BLOCK_SIZE 64
+
146#define IMB_SHA_224_BLOCK_SIZE 64
+
147#define IMB_SHA_256_BLOCK_SIZE 64
+
148#define IMB_SHA_384_BLOCK_SIZE 128
+
149#define IMB_SHA_512_BLOCK_SIZE 128
+
150
+
151#define IMB_MD5_BLOCK_SIZE 64
+
152
+
153#define IMB_KASUMI_KEY_SIZE 16
+
154#define IMB_KASUMI_IV_SIZE 8
+
155#define IMB_KASUMI_BLOCK_SIZE 8
+
156#define IMB_KASUMI_DIGEST_SIZE 4
+
157
+
158#define IMB_ZUC_KEY_LEN_IN_BYTES 16
+
159#define IMB_ZUC_IV_LEN_IN_BYTES 16
+
160#define IMB_ZUC256_KEY_LEN_IN_BYTES 32
+
161#define IMB_ZUC256_IV_LEN_IN_BYTES_MIN 23
+
162#define IMB_ZUC256_IV_LEN_IN_BYTES_MAX 25
+
163#define IMB_ZUC_DIGEST_LEN_IN_BYTES 4
+
164#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN 4
+
165#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX 16
+
166
+
171#define IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE 14
+
172#define IMB_DOCSIS_CRC32_TAG_SIZE 4
+
173
+
178typedef enum {
+
179 IMB_STATUS_BEING_PROCESSED = 0,
+
180 IMB_STATUS_COMPLETED_CIPHER = 1,
+
181 IMB_STATUS_COMPLETED_AUTH = 2,
+
182 IMB_STATUS_COMPLETED = 3,
+
184 IMB_STATUS_INVALID_ARGS = 4,
+
185 IMB_STATUS_INTERNAL_ERROR,
+
186 IMB_STATUS_ERROR
+
187} IMB_STATUS;
+
188
+
192typedef enum {
+
193 IMB_ERR_MIN = 2000,
+
194 IMB_ERR_NULL_MBMGR,
+
195 IMB_ERR_JOB_NULL_SRC,
+
196 IMB_ERR_JOB_NULL_DST,
+
197 IMB_ERR_JOB_NULL_KEY,
+
198 IMB_ERR_JOB_NULL_IV,
+
199 IMB_ERR_JOB_NULL_AUTH,
+
200 IMB_ERR_JOB_NULL_AAD,
+
201 IMB_ERR_JOB_CIPH_LEN,
+
202 IMB_ERR_JOB_AUTH_LEN,
+
203 IMB_ERR_JOB_IV_LEN,
+
204 IMB_ERR_JOB_KEY_LEN,
+
205 IMB_ERR_JOB_AUTH_TAG_LEN,
+
206 IMB_ERR_JOB_AAD_LEN,
+
207 IMB_ERR_JOB_SRC_OFFSET,
+
208 IMB_ERR_JOB_CHAIN_ORDER,
+
209 IMB_ERR_CIPH_MODE,
+
210 IMB_ERR_HASH_ALGO,
+
211 IMB_ERR_JOB_NULL_AUTH_KEY,
+
212 IMB_ERR_JOB_NULL_SGL_CTX,
+
213 IMB_ERR_JOB_NULL_NEXT_IV,
+
214 IMB_ERR_JOB_PON_PLI,
+
215 IMB_ERR_NULL_SRC,
+
216 IMB_ERR_NULL_DST,
+
217 IMB_ERR_NULL_KEY,
+
218 IMB_ERR_NULL_EXP_KEY,
+
219 IMB_ERR_NULL_IV,
+
220 IMB_ERR_NULL_AUTH,
+
221 IMB_ERR_NULL_AAD,
+
222 IMB_ERR_CIPH_LEN,
+
223 IMB_ERR_AUTH_LEN,
+
224 IMB_ERR_IV_LEN,
+
225 IMB_ERR_KEY_LEN,
+
226 IMB_ERR_AUTH_TAG_LEN,
+
227 IMB_ERR_AAD_LEN,
+
228 IMB_ERR_SRC_OFFSET,
+
229 IMB_ERR_NULL_AUTH_KEY,
+
230 IMB_ERR_NULL_CTX,
+
231 IMB_ERR_NO_AESNI_EMU,
+
232 IMB_ERR_JOB_NULL_HMAC_OPAD,
+
233 IMB_ERR_JOB_NULL_HMAC_IPAD,
+
234 IMB_ERR_JOB_NULL_XCBC_K1_EXP,
+
235 IMB_ERR_JOB_NULL_XCBC_K2,
+
236 IMB_ERR_JOB_NULL_XCBC_K3,
+
237 IMB_ERR_JOB_CIPH_DIR,
+
238 IMB_ERR_JOB_NULL_GHASH_INIT_TAG,
+
239 IMB_ERR_MISSING_CPUFLAGS_INIT_MGR,
+
240 IMB_ERR_NULL_JOB,
+
241 IMB_ERR_QUEUE_SPACE,
+
242 IMB_ERR_NULL_BURST,
+
243 IMB_ERR_BURST_SIZE,
+
244 IMB_ERR_BURST_OOO,
+
245 IMB_ERR_SELFTEST,
+
246 IMB_ERR_BURST_SUITE_ID,
+
247 /* add new error types above this comment */
+
248 IMB_ERR_MAX /* don't move this one */
+
249} IMB_ERR;
+
250
+
255#ifdef __ELASTERROR
+
256#if __ELASTERROR > 2000
+
257#error "Library error codes conflict with errno.h - please update IMB_ERR_MIN!"
+
258#endif
+
259#endif
+
260
+
261typedef enum {
+
262 IMB_CIPHER_CBC = 1,
+
263 IMB_CIPHER_CNTR,
+
264 IMB_CIPHER_NULL,
+
265 IMB_CIPHER_DOCSIS_SEC_BPI,
+
266 IMB_CIPHER_GCM,
+
267 IMB_CIPHER_CUSTOM,
+
268 IMB_CIPHER_DES,
+
269 IMB_CIPHER_DOCSIS_DES,
+
270 IMB_CIPHER_CCM,
+
271 IMB_CIPHER_DES3,
+
272 IMB_CIPHER_PON_AES_CNTR,
+
273 IMB_CIPHER_ECB,
+
274 IMB_CIPHER_CNTR_BITLEN,
+
275 IMB_CIPHER_ZUC_EEA3,
+
276 IMB_CIPHER_SNOW3G_UEA2_BITLEN,
+
277 IMB_CIPHER_KASUMI_UEA1_BITLEN,
+
278 IMB_CIPHER_CBCS_1_9,
+
279 IMB_CIPHER_CHACHA20,
+
280 IMB_CIPHER_CHACHA20_POLY1305,
+
281 IMB_CIPHER_CHACHA20_POLY1305_SGL,
+
282 IMB_CIPHER_SNOW_V,
+
283 IMB_CIPHER_SNOW_V_AEAD,
+
284 IMB_CIPHER_GCM_SGL,
+
285 IMB_CIPHER_NUM
+
286} IMB_CIPHER_MODE;
+
287
+
288typedef enum {
+
289 IMB_DIR_ENCRYPT = 1,
+
290 IMB_DIR_DECRYPT
+
291} IMB_CIPHER_DIRECTION;
+
292
+
293typedef enum {
+
294 IMB_AUTH_HMAC_SHA_1 = 1,
+
295 IMB_AUTH_HMAC_SHA_224,
+
296 IMB_AUTH_HMAC_SHA_256,
+
297 IMB_AUTH_HMAC_SHA_384,
+
298 IMB_AUTH_HMAC_SHA_512,
+
299 IMB_AUTH_AES_XCBC,
+
300 IMB_AUTH_MD5,
+
301 IMB_AUTH_NULL,
+
302 IMB_AUTH_AES_GMAC,
+
303 IMB_AUTH_CUSTOM,
+
304 IMB_AUTH_AES_CCM,
+
305 IMB_AUTH_AES_CMAC,
+
306 IMB_AUTH_SHA_1,
+
307 IMB_AUTH_SHA_224,
+
308 IMB_AUTH_SHA_256,
+
309 IMB_AUTH_SHA_384,
+
310 IMB_AUTH_SHA_512,
+
311 IMB_AUTH_AES_CMAC_BITLEN,
+
312 IMB_AUTH_PON_CRC_BIP,
+
313 IMB_AUTH_ZUC_EIA3_BITLEN,
+
314 IMB_AUTH_DOCSIS_CRC32,
+
315 IMB_AUTH_SNOW3G_UIA2_BITLEN,
+
316 IMB_AUTH_KASUMI_UIA1,
+
317 IMB_AUTH_AES_GMAC_128,
+
318 IMB_AUTH_AES_GMAC_192,
+
319 IMB_AUTH_AES_GMAC_256,
+
320 IMB_AUTH_AES_CMAC_256,
+
321 IMB_AUTH_POLY1305,
+
322 IMB_AUTH_CHACHA20_POLY1305,
+
323 IMB_AUTH_CHACHA20_POLY1305_SGL,
+
324 IMB_AUTH_ZUC256_EIA3_BITLEN,
+
325 IMB_AUTH_SNOW_V_AEAD,
+
326 IMB_AUTH_GCM_SGL,
+
327 IMB_AUTH_CRC32_ETHERNET_FCS,
+
328 IMB_AUTH_CRC32_SCTP,
+
329 IMB_AUTH_CRC32_WIMAX_OFDMA_DATA,
+
330 IMB_AUTH_CRC24_LTE_A,
+
331 IMB_AUTH_CRC24_LTE_B,
+
332 IMB_AUTH_CRC16_X25,
+
333 IMB_AUTH_CRC16_FP_DATA,
+
334 IMB_AUTH_CRC11_FP_HEADER,
+
335 IMB_AUTH_CRC10_IUUP_DATA,
+
336 IMB_AUTH_CRC8_WIMAX_OFDMA_HCS,
+
337 IMB_AUTH_CRC7_FP_HEADER,
+
338 IMB_AUTH_CRC6_IUUP_HEADER,
+
339 IMB_AUTH_GHASH,
+
340 IMB_AUTH_NUM
+
341} IMB_HASH_ALG;
+
342
+
343typedef enum {
+
344 IMB_ORDER_CIPHER_HASH = 1,
+
345 IMB_ORDER_HASH_CIPHER
+
346} IMB_CHAIN_ORDER;
+
347
+
348typedef enum {
+
349 IMB_KEY_64_BYTES = 8,
+
350 IMB_KEY_128_BYTES = 16,
+
351 IMB_KEY_192_BYTES = 24,
+
352 IMB_KEY_256_BYTES = 32
+
353} IMB_KEY_SIZE_BYTES;
+
354
+
355typedef enum {
+
356 IMB_SGL_INIT = 0,
+
357 IMB_SGL_UPDATE,
+
358 IMB_SGL_COMPLETE,
+
359 IMB_SGL_ALL
+
360} IMB_SGL_STATE;
+
361
+
365struct IMB_SGL_IOV {
+
366 const void *in;
+
367 void *out;
+
368 uint64_t len;
+
369};
+
370
+
390typedef struct IMB_JOB {
+
391 const void *enc_keys;
+
392 const void *dec_keys;
+
393 uint64_t key_len_in_bytes;
+
394 union {
+
395 const uint8_t *src;
+
398 const struct IMB_SGL_IOV *sgl_io_segs;
+
400 };
+
401 union {
+
402 uint8_t *dst;
+
405 uint64_t num_sgl_io_segs;
+
407 };
+
408 union {
+
409 uint64_t cipher_start_src_offset_in_bytes;
+
411 uint64_t cipher_start_src_offset_in_bits;
+
413 uint64_t cipher_start_offset_in_bits;
+
415 };
+
416 union {
+
417 uint64_t msg_len_to_cipher_in_bytes;
+
419 uint64_t msg_len_to_cipher_in_bits;
+
421 };
+
422 uint64_t hash_start_src_offset_in_bytes;
+
423 union {
+
424 uint64_t msg_len_to_hash_in_bytes;
+
426 uint64_t msg_len_to_hash_in_bits;
+
428 };
+
429 const uint8_t *iv;
+
430 uint64_t iv_len_in_bytes;
+
431 uint8_t *auth_tag_output;
+
432 uint64_t auth_tag_output_len_in_bytes;
+
435 /* Start hash algorithm-specific fields */
+
436 union {
+
437 struct _HMAC_specific_fields {
+
438 const uint8_t *_hashed_auth_key_xor_ipad;
+
441 const uint8_t *_hashed_auth_key_xor_opad;
+
444 } HMAC;
+
445 struct _AES_XCBC_specific_fields {
+
446 const uint32_t *_k1_expanded;
+
448 const uint8_t *_k2;
+
450 const uint8_t *_k3;
+
452 } XCBC;
+
453 struct _AES_CCM_specific_fields {
+
454 const void *aad;
+
456 uint64_t aad_len_in_bytes;
+
457 } CCM;
+
458 struct _AES_CMAC_specific_fields {
+
459 const void *_key_expanded;
+
461 const void *_skey1;
+
462 const void *_skey2;
+
463 } CMAC;
+
464 struct _AES_GCM_specific_fields {
+
465 const void *aad;
+
467 uint64_t aad_len_in_bytes;
+
468 struct gcm_context_data *ctx;
+
470 } GCM;
+
471 struct _ZUC_EIA3_specific_fields {
+
472 const uint8_t *_key;
+
474 const uint8_t *_iv;
+
476 const uint8_t *_iv23;
+
478 } ZUC_EIA3;
+
479 struct _SNOW3G_UIA2_specific_fields {
+
480 const void *_key;
+
482 const void *_iv;
+
484 } SNOW3G_UIA2;
+
485 struct _KASUMI_UIA1_specific_fields {
+
486 const void *_key;
+
488 } KASUMI_UIA1;
+
489 struct _AES_GMAC_specific_fields {
+
490 const struct gcm_key_data *_key;
+
492 const void *_iv;
+
494 uint64_t iv_len_in_bytes;
+
496 } GMAC;
+
497 struct _GHASH_specific_fields {
+
498 const struct gcm_key_data *_key;
+
500 const void *_init_tag;
+
501 } GHASH;
+
502 struct _POLY1305_specific_fields {
+
503 const void *_key;
+
505 } POLY1305;
+
506 struct _CHACHA20_POLY1305_specific_fields {
+
507 const void *aad;
+
509 uint64_t aad_len_in_bytes;
+
511 struct chacha20_poly1305_context_data *ctx;
+
513 } CHACHA20_POLY1305;
+
514 struct _SNOW_V_AEAD_specific_fields {
+
515 const void *aad;
+
517 uint64_t aad_len_in_bytes;
+
519 void *reserved;
+
521 } SNOW_V_AEAD;
+
522 } u;
+
524 IMB_STATUS status;
+
525 IMB_CIPHER_MODE cipher_mode;
+
526 IMB_CIPHER_DIRECTION cipher_direction;
+
527 IMB_HASH_ALG hash_alg;
+
528 IMB_CHAIN_ORDER chain_order;
+
531 void *user_data;
+
532 void *user_data2;
+
534 int (*cipher_func)(struct IMB_JOB *);
+
536 int (*hash_func)(struct IMB_JOB *);
+
539 IMB_SGL_STATE sgl_state;
+
543 union {
+
544 struct _CBCS_specific_fields {
+
545 void *next_iv;
+
547 } CBCS;
+
548 } cipher_fields;
+
550 void *suite_id[4];
+
551 uint32_t session_id;
+
552} IMB_JOB;
+
553
+
554
+
555/* KASUMI */
+
556
+
557/* 64 precomputed words for key schedule */
+
558#define KASUMI_KEY_SCHEDULE_SIZE 64
+
559
+
563typedef struct kasumi_key_sched_s {
+
565 uint16_t sk16[KASUMI_KEY_SCHEDULE_SIZE];
+
566 uint16_t msk16[KASUMI_KEY_SCHEDULE_SIZE];
+
567} kasumi_key_sched_t;
+
568
+
569/* GCM data structures */
+
570#define IMB_GCM_BLOCK_LEN 16
+
571
+
577struct gcm_context_data {
+
578 uint8_t aad_hash[IMB_GCM_BLOCK_LEN];
+
579 uint64_t aad_length;
+
580 uint64_t in_length;
+
581 uint8_t partial_block_enc_key[IMB_GCM_BLOCK_LEN];
+
582 uint8_t orig_IV[IMB_GCM_BLOCK_LEN];
+
583 uint8_t current_counter[IMB_GCM_BLOCK_LEN];
+
584 uint64_t partial_block_length;
+
585};
+
586#undef IMB_GCM_BLOCK_LEN
+
587
+
591struct chacha20_poly1305_context_data {
+
592 uint64_t hash[3];
+
593 uint64_t aad_len;
+
594 uint64_t hash_len;
+
595 uint8_t last_ks[64];
+
596 uint8_t poly_key[32];
+
597 uint8_t poly_scratch[16];
+
598 uint64_t last_block_count;
+
599 uint64_t remain_ks_bytes;
+
601 uint64_t remain_ct_bytes;
+
604 uint8_t IV[12];
+
605};
+
606
+
611#define IMB_MAX_TAG_LEN (16)
+
612
+
622#define IMB_GCM_IV_DATA_LEN (12)
+
623
+
624#define IMB_GCM_128_KEY_LEN (16)
+
625#define IMB_GCM_192_KEY_LEN (24)
+
626#define IMB_GCM_256_KEY_LEN (32)
+
627
+
628#define IMB_GCM_ENC_KEY_LEN 16
+
629#define IMB_GCM_KEY_SETS (15)
+
636#ifdef __WIN32
+
637__declspec(align(64))
+
638#endif /* WIN32 */
+
639struct gcm_key_data {
+
640 uint8_t expanded_keys[IMB_GCM_ENC_KEY_LEN * IMB_GCM_KEY_SETS];
+
641 union {
+
643 struct {
+
649 uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 8];
+
658 uint8_t shifted_hkey_k[IMB_GCM_ENC_KEY_LEN * 8];
+
659 } sse_avx;
+
660 struct {
+
666 uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 8];
+
667 } avx2_avx512;
+
668 struct {
+
679 uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 32 * 2];
+
680 } vaes_avx512;
+
681 } ghash_keys;
+
682}
+
683#ifdef LINUX
+
684__attribute__((aligned(64)));
+
685#else
+
686;
+
687#endif
+
688
+
689#undef IMB_GCM_ENC_KEY_LEN
+
690#undef IMB_GCM_KEY_SETS
+
691
+
692/* API data type definitions */
+
693struct IMB_MGR;
+
694
+
695typedef void (*init_mb_mgr_t)(struct IMB_MGR *);
+
696typedef IMB_JOB *(*get_next_job_t)(struct IMB_MGR *);
+
697typedef IMB_JOB *(*submit_job_t)(struct IMB_MGR *);
+
698typedef IMB_JOB *(*get_completed_job_t)(struct IMB_MGR *);
+
699typedef IMB_JOB *(*flush_job_t)(struct IMB_MGR *);
+
700typedef uint32_t (*queue_size_t)(struct IMB_MGR *);
+
701typedef uint32_t (*burst_fn_t)(struct IMB_MGR *,
+
702 const uint32_t,
+
703 struct IMB_JOB **);
+
704typedef uint32_t (*submit_cipher_burst_t)(struct IMB_MGR *,
+
705 struct IMB_JOB *,
+
706 const uint32_t,
+
707 const IMB_CIPHER_MODE cipher,
+
708 const IMB_CIPHER_DIRECTION dir,
+
709 const IMB_KEY_SIZE_BYTES key_size);
+
710typedef uint32_t (*submit_hash_burst_t)(struct IMB_MGR *,
+
711 struct IMB_JOB *,
+
712 const uint32_t,
+
713 const IMB_HASH_ALG hash);
+
714typedef void (*keyexp_t)(const void *, void *, void *);
+
715typedef void (*cmac_subkey_gen_t)(const void *, void *, void *);
+
716typedef void (*hash_one_block_t)(const void *, void *);
+
717typedef void (*hash_fn_t)(const void *, const uint64_t, void *);
+
718typedef void (*xcbc_keyexp_t)(const void *, void *, void *, void *);
+
719typedef int (*des_keysched_t)(uint64_t *, const void *);
+
720typedef void (*aes_cfb_t)(void *, const void *, const void *, const void *,
+
721 uint64_t);
+
722typedef void (*aes_gcm_enc_dec_t)(const struct gcm_key_data *,
+
723 struct gcm_context_data *,
+
724 uint8_t *, uint8_t const *, uint64_t,
+
725 const uint8_t *, uint8_t const *, uint64_t,
+
726 uint8_t *, uint64_t);
+
727typedef void (*aes_gcm_enc_dec_iv_t)(const struct gcm_key_data *,
+
728 struct gcm_context_data *, uint8_t *,
+
729 uint8_t const *, const uint64_t,
+
730 const uint8_t *, uint8_t const *,
+
731 const uint64_t, uint8_t *,
+
732 const uint64_t, const uint64_t);
+
733typedef void (*aes_gcm_init_t)(const struct gcm_key_data *,
+
734 struct gcm_context_data *,
+
735 const uint8_t *, uint8_t const *, uint64_t);
+
736typedef void (*aes_gcm_init_var_iv_t)(const struct gcm_key_data *,
+
737 struct gcm_context_data *,
+
738 const uint8_t *, const uint64_t,
+
739 const uint8_t *, const uint64_t);
+
740typedef void (*aes_gcm_enc_dec_update_t)(const struct gcm_key_data *,
+
741 struct gcm_context_data *,
+
742 uint8_t *, const uint8_t *, uint64_t);
+
743typedef void (*aes_gcm_enc_dec_finalize_t)(const struct gcm_key_data *,
+
744 struct gcm_context_data *,
+
745 uint8_t *, uint64_t);
+
746typedef void (*aes_gcm_precomp_t)(struct gcm_key_data *);
+
747typedef void (*aes_gcm_pre_t)(const void *, struct gcm_key_data *);
+
748
+
749typedef void (*aes_gmac_init_t)(const struct gcm_key_data *,
+
750 struct gcm_context_data *,
+
751 const uint8_t *, const uint64_t);
+
752typedef void (*aes_gmac_update_t)(const struct gcm_key_data *,
+
753 struct gcm_context_data *,
+
754 const uint8_t *, const uint64_t);
+
755typedef void (*aes_gmac_finalize_t)(const struct gcm_key_data *,
+
756 struct gcm_context_data *,
+
757 uint8_t *, const uint64_t);
+
758
+
759typedef void (*chacha_poly_init_t)(const void *,
+
760 struct chacha20_poly1305_context_data *,
+
761 const void *, const void *, const uint64_t);
+
762typedef void (*chacha_poly_enc_dec_update_t)(const void *,
+
763 struct chacha20_poly1305_context_data *,
+
764 void *, const void *, const uint64_t);
+
765typedef void (*chacha_poly_finalize_t)(struct chacha20_poly1305_context_data *,
+
766 void *, const uint64_t);
+
767typedef void (*ghash_t)(const struct gcm_key_data *, const void *,
+
768 const uint64_t, void *, const uint64_t);
+
769
+
770typedef void (*zuc_eea3_1_buffer_t)(const void *, const void *, const void *,
+
771 void *, const uint32_t);
+
772
+
773typedef void (*zuc_eea3_4_buffer_t)(const void * const *, const void * const *,
+
774 const void * const *, void **,
+
775 const uint32_t *);
+
776
+
777typedef void (*zuc_eea3_n_buffer_t)(const void * const *, const void * const *,
+
778 const void * const *, void **,
+
779 const uint32_t *, const uint32_t);
+
780
+
781typedef void (*zuc_eia3_1_buffer_t)(const void *, const void *, const void *,
+
782 const uint32_t, uint32_t *);
+
783
+
784typedef void (*zuc_eia3_n_buffer_t)(const void * const *, const void * const *,
+
785 const void * const *,
+
786 const uint32_t *, uint32_t **,
+
787 const uint32_t);
+
788
+
789
+
790typedef void (*kasumi_f8_1_buffer_t)(const kasumi_key_sched_t *,
+
791 const uint64_t, const void *, void *,
+
792 const uint32_t);
+
793typedef void (*kasumi_f8_1_buffer_bit_t)(const kasumi_key_sched_t *,
+
794 const uint64_t, const void *,
+
795 void *,
+
796 const uint32_t, const uint32_t);
+
797typedef void (*kasumi_f8_2_buffer_t)(const kasumi_key_sched_t *,
+
798 const uint64_t, const uint64_t,
+
799 const void *, void *,
+
800 const uint32_t,
+
801 const void *, void *,
+
802 const uint32_t);
+
803typedef void (*kasumi_f8_3_buffer_t)(const kasumi_key_sched_t *,
+
804 const uint64_t, const uint64_t,
+
805 const uint64_t,
+
806 const void *, void *,
+
807 const void *, void *,
+
808 const void *, void *,
+
809 const uint32_t);
+
810typedef void (*kasumi_f8_4_buffer_t)(const kasumi_key_sched_t *,
+
811 const uint64_t, const uint64_t,
+
812 const uint64_t, const uint64_t,
+
813 const void *, void *,
+
814 const void *, void *,
+
815 const void *, void *,
+
816 const void *, void *,
+
817 const uint32_t);
+
818typedef void (*kasumi_f8_n_buffer_t)(const kasumi_key_sched_t *,
+
819 const uint64_t *, const void * const *,
+
820 void **, const uint32_t *,
+
821 const uint32_t);
+
822typedef void (*kasumi_f9_1_buffer_user_t)(const kasumi_key_sched_t *,
+
823 const uint64_t, const void *,
+
824 const uint32_t, void *,
+
825 const uint32_t);
+
826typedef void (*kasumi_f9_1_buffer_t)(const kasumi_key_sched_t *,
+
827 const void *,
+
828 const uint32_t, void *);
+
829typedef int (*kasumi_init_f8_key_sched_t)(const void *,
+
830 kasumi_key_sched_t *);
+
831typedef int (*kasumi_init_f9_key_sched_t)(const void *,
+
832 kasumi_key_sched_t *);
+
833typedef size_t (*kasumi_key_sched_size_t)(void);
+
834
+
835
+
839typedef struct snow3g_key_schedule_s {
+
840 /* KEY */
+
841 uint32_t k[4];
+
842} snow3g_key_schedule_t;
+
843
+
844typedef void (*snow3g_f8_1_buffer_t)(const snow3g_key_schedule_t *,
+
845 const void *, const void *,
+
846 void *, const uint32_t);
+
847
+
848typedef void (*snow3g_f8_1_buffer_bit_t)(const snow3g_key_schedule_t *,
+
849 const void *, const void *, void *,
+
850 const uint32_t, const uint32_t);
+
851
+
852typedef void (*snow3g_f8_2_buffer_t)(const snow3g_key_schedule_t *,
+
853 const void *, const void *,
+
854 const void *, void *, const uint32_t,
+
855 const void *, void *, const uint32_t);
+
856
+
857typedef void (*snow3g_f8_4_buffer_t)(const snow3g_key_schedule_t *,
+
858 const void *, const void *, const void *,
+
859 const void *, const void *, void *,
+
860 const uint32_t, const void *, void *,
+
861 const uint32_t, const void *, void *,
+
862 const uint32_t, const void *, void *,
+
863 const uint32_t);
+
864
+
865typedef void (*snow3g_f8_8_buffer_t)(const snow3g_key_schedule_t *,
+
866 const void *, const void *, const void *,
+
867 const void *, const void *, const void *,
+
868 const void *, const void *, const void *,
+
869 void *, const uint32_t, const void *,
+
870 void *, const uint32_t, const void *,
+
871 void *, const uint32_t, const void *,
+
872 void *, const uint32_t, const void *,
+
873 void *, const uint32_t, const void *,
+
874 void *, const uint32_t, const void *,
+
875 void *, const uint32_t, const void *,
+
876 void *, const uint32_t);
+
877
+
878typedef void
+
879(*snow3g_f8_8_buffer_multikey_t)(const snow3g_key_schedule_t * const [],
+
880 const void * const [], const void * const [],
+
881 void *[], const uint32_t[]);
+
882
+
883typedef void (*snow3g_f8_n_buffer_t)(const snow3g_key_schedule_t *,
+
884 const void * const [],
+
885 const void * const [],
+
886 void *[], const uint32_t[],
+
887 const uint32_t);
+
888
+
889typedef void
+
890(*snow3g_f8_n_buffer_multikey_t)(const snow3g_key_schedule_t * const [],
+
891 const void * const [],
+
892 const void * const [],
+
893 void *[], const uint32_t[],
+
894 const uint32_t);
+
895
+
896typedef void (*snow3g_f9_1_buffer_t)(const snow3g_key_schedule_t *,
+
897 const void *, const void *,
+
898 const uint64_t, void *);
+
899
+
900typedef int (*snow3g_init_key_sched_t)(const void *,
+
901 snow3g_key_schedule_t *);
+
902
+
903typedef size_t (*snow3g_key_sched_size_t)(void);
+
904
+
905typedef uint32_t (*hec_32_t)(const uint8_t *);
+
906typedef uint64_t (*hec_64_t)(const uint8_t *);
+
907
+
908typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t);
+
909
+
910typedef void (*aes_ecb_quic_t)(const void *, const void *,
+
911 void *out, uint64_t);
+
912
+
913/* Multi-buffer manager flags passed to alloc_mb_mgr() */
+
914
+
915#define IMB_FLAG_SHANI_OFF (1ULL << 0)
+
916#define IMB_FLAG_AESNI_OFF (1ULL << 1)
+
917#define IMB_FLAG_GFNI_OFF (1ULL << 2)
+
926#define IMB_FEATURE_SHANI (1ULL << 0)
+
927#define IMB_FEATURE_AESNI (1ULL << 1)
+
928#define IMB_FEATURE_PCLMULQDQ (1ULL << 2)
+
929#define IMB_FEATURE_CMOV (1ULL << 3)
+
930#define IMB_FEATURE_SSE4_2 (1ULL << 4)
+
931#define IMB_FEATURE_AVX (1ULL << 5)
+
932#define IMB_FEATURE_AVX2 (1ULL << 6)
+
933#define IMB_FEATURE_AVX512F (1ULL << 7)
+
934#define IMB_FEATURE_AVX512DQ (1ULL << 8)
+
935#define IMB_FEATURE_AVX512CD (1ULL << 9)
+
936#define IMB_FEATURE_AVX512BW (1ULL << 10)
+
937#define IMB_FEATURE_AVX512VL (1ULL << 11)
+
938#define IMB_FEATURE_AVX512_SKX (IMB_FEATURE_AVX512F | IMB_FEATURE_AVX512DQ | \
+
939 IMB_FEATURE_AVX512CD | IMB_FEATURE_AVX512BW | \
+
940 IMB_FEATURE_AVX512VL)
+
941#define IMB_FEATURE_VAES (1ULL << 12)
+
942#define IMB_FEATURE_VPCLMULQDQ (1ULL << 13)
+
943#define IMB_FEATURE_SAFE_DATA (1ULL << 14)
+
944#define IMB_FEATURE_SAFE_PARAM (1ULL << 15)
+
945#define IMB_FEATURE_GFNI (1ULL << 16)
+
946#define IMB_FEATURE_AVX512_IFMA (1ULL << 17)
+
947#define IMB_FEATURE_BMI2 (1ULL << 18)
+
948#define IMB_FEATURE_AESNI_EMU (1ULL << 19)
+
949#define IMB_FEATURE_SELF_TEST (1ULL << 20) /* self-test feature present */
+
950#define IMB_FEATURE_SELF_TEST_PASS (1ULL << 21) /* self-test passed */
+
951#define IMB_FEATURE_AVX_IFMA (1ULL << 22)
+
952
+
956#define IMB_CPUFLAGS_NO_AESNI (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV)
+
957#define IMB_CPUFLAGS_SSE (IMB_CPUFLAGS_NO_AESNI | IMB_FEATURE_AESNI | \
+
958 IMB_FEATURE_PCLMULQDQ)
+
959#define IMB_CPUFLAGS_SSE_T2 (IMB_CPUFLAGS_SSE | IMB_FEATURE_SHANI)
+
960#define IMB_CPUFLAGS_SSE_T3 (IMB_CPUFLAGS_SSE_T2 | IMB_FEATURE_GFNI)
+
961#define IMB_CPUFLAGS_AVX (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX)
+
962#define IMB_CPUFLAGS_AVX2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_AVX2 | \
+
963 IMB_FEATURE_BMI2)
+
964#define IMB_CPUFLAGS_AVX512 (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX)
+
965#define IMB_CPUFLAGS_AVX512_T2 (IMB_CPUFLAGS_AVX512 | IMB_FEATURE_VAES | \
+
966 IMB_FEATURE_VPCLMULQDQ | IMB_FEATURE_GFNI | \
+
967 IMB_FEATURE_AVX512_IFMA | IMB_FEATURE_SHANI)
+
968#define IMB_CPUFLAGS_AVX2_T2 (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_SHANI | \
+
969 IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ | \
+
970 IMB_FEATURE_GFNI)
+
971#define IMB_CPUFLAGS_AVX_T2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_SHANI | \
+
972 IMB_FEATURE_GFNI)
+
973#define IMB_CPUFLAGS_AVX2_T3 (IMB_CPUFLAGS_AVX2_T2 | IMB_FEATURE_AVX_IFMA)
+
974
+
975/* TOP LEVEL (IMB_MGR) Data structure fields */
+
976
+
977#define IMB_MAX_BURST_SIZE 128
+
978#define IMB_MAX_JOBS (IMB_MAX_BURST_SIZE * 2)
+
979
+
980typedef struct IMB_MGR {
+
981
+
982 uint64_t flags;
+
983 uint64_t features;
+
985 uint64_t reserved[5];
+
986 uint32_t used_arch;
+
988 int imb_errno;
+
996 get_next_job_t get_next_job;
+
997 submit_job_t submit_job;
+
998 submit_job_t submit_job_nocheck;
+
999 get_completed_job_t get_completed_job;
+
1000 flush_job_t flush_job;
+
1001 queue_size_t queue_size;
+
1002 keyexp_t keyexp_128;
+
1003 keyexp_t keyexp_192;
+
1004 keyexp_t keyexp_256;
+
1005 cmac_subkey_gen_t cmac_subkey_gen_128;
+
1006 xcbc_keyexp_t xcbc_keyexp;
+
1007 des_keysched_t des_key_sched;
+
1008 hash_one_block_t sha1_one_block;
+
1009 hash_one_block_t sha224_one_block;
+
1010 hash_one_block_t sha256_one_block;
+
1011 hash_one_block_t sha384_one_block;
+
1012 hash_one_block_t sha512_one_block;
+
1013 hash_one_block_t md5_one_block;
+
1014 hash_fn_t sha1;
+
1015 hash_fn_t sha224;
+
1016 hash_fn_t sha256;
+
1017 hash_fn_t sha384;
+
1018 hash_fn_t sha512;
+
1019 aes_cfb_t aes128_cfb_one;
+
1020
+
1021 aes_gcm_enc_dec_t gcm128_enc;
+
1022 aes_gcm_enc_dec_t gcm192_enc;
+
1023 aes_gcm_enc_dec_t gcm256_enc;
+
1024 aes_gcm_enc_dec_t gcm128_dec;
+
1025 aes_gcm_enc_dec_t gcm192_dec;
+
1026 aes_gcm_enc_dec_t gcm256_dec;
+
1027 aes_gcm_init_t gcm128_init;
+
1028 aes_gcm_init_t gcm192_init;
+
1029 aes_gcm_init_t gcm256_init;
+
1030 aes_gcm_enc_dec_update_t gcm128_enc_update;
+
1031 aes_gcm_enc_dec_update_t gcm192_enc_update;
+
1032 aes_gcm_enc_dec_update_t gcm256_enc_update;
+
1033 aes_gcm_enc_dec_update_t gcm128_dec_update;
+
1034 aes_gcm_enc_dec_update_t gcm192_dec_update;
+
1035 aes_gcm_enc_dec_update_t gcm256_dec_update;
+
1036 aes_gcm_enc_dec_finalize_t gcm128_enc_finalize;
+
1037 aes_gcm_enc_dec_finalize_t gcm192_enc_finalize;
+
1038 aes_gcm_enc_dec_finalize_t gcm256_enc_finalize;
+
1039 aes_gcm_enc_dec_finalize_t gcm128_dec_finalize;
+
1040 aes_gcm_enc_dec_finalize_t gcm192_dec_finalize;
+
1041 aes_gcm_enc_dec_finalize_t gcm256_dec_finalize;
+
1042 aes_gcm_precomp_t gcm128_precomp;
+
1043 aes_gcm_precomp_t gcm192_precomp;
+
1044 aes_gcm_precomp_t gcm256_precomp;
+
1045 aes_gcm_pre_t gcm128_pre;
+
1046 aes_gcm_pre_t gcm192_pre;
+
1047 aes_gcm_pre_t gcm256_pre;
+
1048
+
1049 zuc_eea3_1_buffer_t eea3_1_buffer;
+
1050 zuc_eea3_4_buffer_t eea3_4_buffer;
+
1051 zuc_eea3_n_buffer_t eea3_n_buffer;
+
1052 zuc_eia3_1_buffer_t eia3_1_buffer;
+
1053
+
1054 kasumi_f8_1_buffer_t f8_1_buffer;
+
1055 kasumi_f8_1_buffer_bit_t f8_1_buffer_bit;
+
1056 kasumi_f8_2_buffer_t f8_2_buffer;
+
1057 kasumi_f8_3_buffer_t f8_3_buffer;
+
1058 kasumi_f8_4_buffer_t f8_4_buffer;
+
1059 kasumi_f8_n_buffer_t f8_n_buffer;
+
1060 kasumi_f9_1_buffer_t f9_1_buffer;
+
1061 kasumi_f9_1_buffer_user_t f9_1_buffer_user;
+
1062 kasumi_init_f8_key_sched_t kasumi_init_f8_key_sched;
+
1063 kasumi_init_f9_key_sched_t kasumi_init_f9_key_sched;
+
1064 kasumi_key_sched_size_t kasumi_key_sched_size;
+
1065
+
1066 snow3g_f8_1_buffer_bit_t snow3g_f8_1_buffer_bit;
+
1067 snow3g_f8_1_buffer_t snow3g_f8_1_buffer;
+
1068 snow3g_f8_2_buffer_t snow3g_f8_2_buffer;
+
1069 snow3g_f8_4_buffer_t snow3g_f8_4_buffer;
+
1070 snow3g_f8_8_buffer_t snow3g_f8_8_buffer;
+
1071 snow3g_f8_n_buffer_t snow3g_f8_n_buffer;
+
1072 snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey;
+
1073 snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey;
+
1074 snow3g_f9_1_buffer_t snow3g_f9_1_buffer;
+
1075 snow3g_init_key_sched_t snow3g_init_key_sched;
+
1076 snow3g_key_sched_size_t snow3g_key_sched_size;
+
1077
+
1078 ghash_t ghash;
+
1079 zuc_eia3_n_buffer_t eia3_n_buffer;
+
1080 aes_gcm_init_var_iv_t gcm128_init_var_iv;
+
1081 aes_gcm_init_var_iv_t gcm192_init_var_iv;
+
1082 aes_gcm_init_var_iv_t gcm256_init_var_iv;
+
1083
+
1084 aes_gmac_init_t gmac128_init;
+
1085 aes_gmac_init_t gmac192_init;
+
1086 aes_gmac_init_t gmac256_init;
+
1087 aes_gmac_update_t gmac128_update;
+
1088 aes_gmac_update_t gmac192_update;
+
1089 aes_gmac_update_t gmac256_update;
+
1090 aes_gmac_finalize_t gmac128_finalize;
+
1091 aes_gmac_finalize_t gmac192_finalize;
+
1092 aes_gmac_finalize_t gmac256_finalize;
+
1093 hec_32_t hec_32;
+
1094 hec_64_t hec_64;
+
1095 cmac_subkey_gen_t cmac_subkey_gen_256;
+
1096 aes_gcm_pre_t ghash_pre;
+
1097 crc32_fn_t crc32_ethernet_fcs;
+
1098 crc32_fn_t crc16_x25;
+
1099 crc32_fn_t crc32_sctp;
+
1100 crc32_fn_t crc24_lte_a;
+
1101 crc32_fn_t crc24_lte_b;
+
1102 crc32_fn_t crc16_fp_data;
+
1103 crc32_fn_t crc11_fp_header;
+
1104 crc32_fn_t crc7_fp_header;
+
1105 crc32_fn_t crc10_iuup_data;
+
1106 crc32_fn_t crc6_iuup_header;
+
1107 crc32_fn_t crc32_wimax_ofdma_data;
+
1108 crc32_fn_t crc8_wimax_ofdma_hcs;
+
1109
+
1110 chacha_poly_init_t chacha20_poly1305_init;
+
1111 chacha_poly_enc_dec_update_t chacha20_poly1305_enc_update;
+
1112 chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update;
+
1113 chacha_poly_finalize_t chacha20_poly1305_finalize;
+
1114
+
1115 burst_fn_t get_next_burst;
+
1116 burst_fn_t submit_burst;
+
1117 burst_fn_t submit_burst_nocheck;
+
1118 burst_fn_t flush_burst;
+
1119 submit_cipher_burst_t submit_cipher_burst;
+
1120 submit_cipher_burst_t submit_cipher_burst_nocheck;
+
1121 submit_hash_burst_t submit_hash_burst;
+
1122 submit_hash_burst_t submit_hash_burst_nocheck;
+
1123 aes_cfb_t aes256_cfb_one;
+
1124
+
1125 aes_ecb_quic_t aes_ecb_128_quic;
+
1126 aes_ecb_quic_t aes_ecb_256_quic;
+
1127
+
1128 void (*set_suite_id)(struct IMB_MGR *, IMB_JOB *);
+
1129
+
1130 /* in-order scheduler fields */
+
1131 int earliest_job;
+
1132 int next_job;
+
1133 IMB_JOB jobs[IMB_MAX_JOBS];
+
1134
+
1135 /* out of order managers */
+
1136 void *aes128_ooo;
+
1137 void *aes192_ooo;
+
1138 void *aes256_ooo;
+
1139 void *docsis128_sec_ooo;
+
1140 void *docsis128_crc32_sec_ooo;
+
1141 void *docsis256_sec_ooo;
+
1142 void *docsis256_crc32_sec_ooo;
+
1143 void *des_enc_ooo;
+
1144 void *des_dec_ooo;
+
1145 void *des3_enc_ooo;
+
1146 void *des3_dec_ooo;
+
1147 void *docsis_des_enc_ooo;
+
1148 void *docsis_des_dec_ooo;
+
1149
+
1150 void *hmac_sha_1_ooo;
+
1151 void *hmac_sha_224_ooo;
+
1152 void *hmac_sha_256_ooo;
+
1153 void *hmac_sha_384_ooo;
+
1154 void *hmac_sha_512_ooo;
+
1155 void *hmac_md5_ooo;
+
1156 void *aes_xcbc_ooo;
+
1157 void *aes_ccm_ooo;
+
1158 void *aes_cmac_ooo;
+
1159 void *zuc_eea3_ooo;
+
1160 void *zuc_eia3_ooo;
+
1161 void *aes128_cbcs_ooo;
+
1162 void *zuc256_eea3_ooo;
+
1163 void *zuc256_eia3_ooo;
+
1164 void *aes256_ccm_ooo;
+
1165 void *aes256_cmac_ooo;
+
1166 void *snow3g_uea2_ooo;
+
1167 void *snow3g_uia2_ooo;
+
1168 void *sha_1_ooo;
+
1169 void *sha_224_ooo;
+
1170 void *sha_256_ooo;
+
1171 void *sha_384_ooo;
+
1172 void *sha_512_ooo;
+
1173 void *end_ooo; /* add new out-of-order managers above this line */
+
1174} IMB_MGR;
+
1175
+
1186IMB_DLL_EXPORT const char *imb_get_version_str(void);
+
1187
+
1196IMB_DLL_EXPORT unsigned imb_get_version(void);
+
1197
+
1198
+
1206IMB_DLL_EXPORT int imb_get_errno(IMB_MGR *mb_mgr);
+
1207
+
1215IMB_DLL_EXPORT const char *imb_get_strerror(int errnum);
+
1216
+
1242IMB_DLL_EXPORT IMB_MGR *alloc_mb_mgr(uint64_t flags);
+
1243
+
1250IMB_DLL_EXPORT void free_mb_mgr(IMB_MGR *ptr);
+
1251
+
1257IMB_DLL_EXPORT size_t imb_get_mb_mgr_size(void);
+
1258
+
1281IMB_DLL_EXPORT IMB_MGR *imb_set_pointers_mb_mgr(void *ptr, const uint64_t flags,
+
1282 const unsigned reset_mgr);
+
1283
+
1290IMB_DLL_EXPORT uint64_t imb_get_feature_flags(void);
+
1291
+
1302IMB_DLL_EXPORT void init_mb_mgr_avx(IMB_MGR *state);
+
1306IMB_DLL_EXPORT void init_mb_mgr_avx2(IMB_MGR *state);
+
1310IMB_DLL_EXPORT void init_mb_mgr_avx512(IMB_MGR *state);
+
1314IMB_DLL_EXPORT void init_mb_mgr_sse(IMB_MGR *state);
+
1315
+
1316
+
1326IMB_DLL_EXPORT IMB_JOB *submit_job_avx(IMB_MGR *state);
+
1330IMB_DLL_EXPORT IMB_JOB *submit_job_avx2(IMB_MGR *state);
+
1334IMB_DLL_EXPORT IMB_JOB *submit_job_avx512(IMB_MGR *state);
+
1338IMB_DLL_EXPORT IMB_JOB *submit_job_sse(IMB_MGR *state);
+
1339
+
1349IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx(IMB_MGR *state);
+
1353IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2(IMB_MGR *state);
+
1357IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512(IMB_MGR *state);
+
1361IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse(IMB_MGR *state);
+
1362
+
1370IMB_DLL_EXPORT IMB_JOB *flush_job_avx(IMB_MGR *state);
+
1374IMB_DLL_EXPORT IMB_JOB *flush_job_avx2(IMB_MGR *state);
+
1378IMB_DLL_EXPORT IMB_JOB *flush_job_avx512(IMB_MGR *state);
+
1382IMB_DLL_EXPORT IMB_JOB *flush_job_sse(IMB_MGR *state);
+
1383
+
1391IMB_DLL_EXPORT uint32_t queue_size_avx(IMB_MGR *state);
+
1395IMB_DLL_EXPORT uint32_t queue_size_avx2(IMB_MGR *state);
+
1399IMB_DLL_EXPORT uint32_t queue_size_avx512(IMB_MGR *state);
+
1403IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state);
+
1404
+
1412IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx(IMB_MGR *state);
+
1416IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2(IMB_MGR *state);
+
1420IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512(IMB_MGR *state);
+
1424IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse(IMB_MGR *state);
+
1425
+
1433IMB_DLL_EXPORT IMB_JOB *get_next_job_avx(IMB_MGR *state);
+
1437IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2(IMB_MGR *state);
+
1441IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512(IMB_MGR *state);
+
1445IMB_DLL_EXPORT IMB_JOB *get_next_job_sse(IMB_MGR *state);
+
1446
+
1455IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch);
+
1456
+
1457/*
+
1458 * Wrapper macros to call arch API's set up
+
1459 * at init phase of multi-buffer manager.
+
1460 *
+
1461 * For example, after calling init_mb_mgr_sse(&mgr)
+
1462 * The 'mgr' structure be set up so that:
+
1463 * mgr.get_next_job will point to get_next_job_sse(),
+
1464 * mgr.submit_job will point to submit_job_sse(),
+
1465 * mgr.submit_job_nocheck will point to submit_job_nocheck_sse(),
+
1466 * mgr.get_completed_job will point to get_completed_job_sse(),
+
1467 * mgr.flush_job will point to flush_job_sse(),
+
1468 * mgr.queue_size will point to queue_size_sse()
+
1469 * mgr.keyexp_128 will point to aes_keyexp_128_sse()
+
1470 * mgr.keyexp_192 will point to aes_keyexp_192_sse()
+
1471 * mgr.keyexp_256 will point to aes_keyexp_256_sse()
+
1472 * etc.
+
1473 *
+
1474 * Direct use of arch API's may result in better performance.
+
1475 * Using below indirect interface may produce slightly worse performance but
+
1476 * it can simplify application implementation.
+
1477 * The test app provides example of using the indirect interface.
+
1478 */
+
1479
+
1487#define IMB_GET_NEXT_JOB(_mgr) ((_mgr)->get_next_job((_mgr)))
+
1488
+
1498#define IMB_SUBMIT_JOB(_mgr) ((_mgr)->submit_job((_mgr)))
+
1499
+
1509#define IMB_SUBMIT_JOB_NOCHECK(_mgr) ((_mgr)->submit_job_nocheck((_mgr)))
+
1510
+
1518#define IMB_GET_COMPLETED_JOB(_mgr) ((_mgr)->get_completed_job((_mgr)))
+
1519
+
1527#define IMB_FLUSH_JOB(_mgr) ((_mgr)->flush_job((_mgr)))
+
1528
+
1536#define IMB_QUEUE_SIZE(_mgr) ((_mgr)->queue_size((_mgr)))
+
1537
+
1550#define IMB_GET_NEXT_BURST(_mgr, _n_jobs, _jobs) \
+
1551 ((_mgr)->get_next_burst((_mgr), (_n_jobs), (_jobs)))
+
1552
+
1569#define IMB_SUBMIT_BURST(_mgr, _n_jobs, _jobs) \
+
1570 ((_mgr)->submit_burst((_mgr), (_n_jobs), (_jobs)))
+
1571
+
1586#define IMB_SUBMIT_BURST_NOCHECK(_mgr, _n_jobs, _jobs) \
+
1587 ((_mgr)->submit_burst_nocheck((_mgr), (_n_jobs), (_jobs)))
+
1588
+
1598#define IMB_FLUSH_BURST(_mgr, _max_jobs, _jobs) \
+
1599 ((_mgr)->flush_burst((_mgr), (_max_jobs), (_jobs)))
+
1600
+
1613#define IMB_SUBMIT_CIPHER_BURST(_mgr, _jobs, _n_jobs, _cipher, \
+
1614 _dir, _key_size) \
+
1615 ((_mgr)->submit_cipher_burst((_mgr), (_jobs), (_n_jobs), \
+
1616 (_cipher), (_dir), (_key_size)))
+
1631#define IMB_SUBMIT_CIPHER_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _cipher, \
+
1632 _dir, _key_size) \
+
1633 ((_mgr)->submit_cipher_burst_nocheck((_mgr), (_jobs), (_n_jobs),\
+
1634 (_cipher), (_dir), (_key_size)))
+
1645#define IMB_SUBMIT_HASH_BURST(_mgr, _jobs, _n_jobs, _hash) \
+
1646 ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash)))
+
1647
+
1660#define IMB_SUBMIT_HASH_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _hash) \
+
1661 ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash)))
+
1662
+
1663/* Key expansion and generation API's */
+
1664
+
1673#define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key) \
+
1674 ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key)))
+
1683#define IMB_AES_KEYEXP_192(_mgr, _key, _enc_exp_key, _dec_exp_key) \
+
1684 ((_mgr)->keyexp_192((_key), (_enc_exp_key), (_dec_exp_key)))
+
1693#define IMB_AES_KEYEXP_256(_mgr, _key, _enc_exp_key, _dec_exp_key) \
+
1694 ((_mgr)->keyexp_256((_key), (_enc_exp_key), (_dec_exp_key)))
+
1695
+
1704#define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _exp_key, _key1, _key2) \
+
1705 ((_mgr)->cmac_subkey_gen_128((_exp_key), (_key1), (_key2)))
+
1706
+
1715#define IMB_AES_CMAC_SUBKEY_GEN_256(_mgr, _exp_key, _key1, _key2) \
+
1716 ((_mgr)->cmac_subkey_gen_256((_exp_key), (_key1), (_key2)))
+
1717
+
1727#define IMB_AES_XCBC_KEYEXP(_mgr, _key, _exp_key, _exp_key2, _exp_key3) \
+
1728 ((_mgr)->xcbc_keyexp((_key), (_exp_key), (_exp_key2), (_exp_key3)))
+
1729
+
1730#define IMB_DES_KEYSCHED(_mgr, _exp_key, _key) \
+
1731 ((_mgr)->des_key_sched((_exp_key), (_key)))
+
1732
+
1733/* Hash API's */
+
1734
+
1742#define IMB_SHA1_ONE_BLOCK(_mgr, _src, _tag) \
+
1743 ((_mgr)->sha1_one_block((_src), (_tag)))
+
1744
+
1753#define IMB_SHA1(_mgr, _src, _length, _tag) \
+
1754 ((_mgr)->sha1((_src), (_length), (_tag)))
+
1762#define IMB_SHA224_ONE_BLOCK(_mgr, _src, _tag) \
+
1763 ((_mgr)->sha224_one_block((_src), (_tag)))
+
1764
+
1773#define IMB_SHA224(_mgr, _src, _length, _tag) \
+
1774 ((_mgr)->sha224((_src), (_length), (_tag)))
+
1782#define IMB_SHA256_ONE_BLOCK(_mgr, _src, _tag) \
+
1783 ((_mgr)->sha256_one_block((_src), (_tag)))
+
1792#define IMB_SHA256(_mgr, _src, _length, _tag) \
+
1793 ((_mgr)->sha256((_src), (_length), (_tag)))
+
1801#define IMB_SHA384_ONE_BLOCK(_mgr, _src, _tag) \
+
1802 ((_mgr)->sha384_one_block((_src), (_tag)))
+
1811#define IMB_SHA384(_mgr, _src, _length, _tag) \
+
1812 ((_mgr)->sha384((_src), (_length), (_tag)))
+
1820#define IMB_SHA512_ONE_BLOCK(_mgr, _src, _tag) \
+
1821 ((_mgr)->sha512_one_block((_src), (_tag)))
+
1830#define IMB_SHA512(_mgr, _src, _length, _tag) \
+
1831 ((_mgr)->sha512((_src), (_length), (_tag)))
+
1839#define IMB_MD5_ONE_BLOCK(_mgr, _src, _tag) \
+
1840 ((_mgr)->md5_one_block((_src), (_tag)))
+
1841
+
1855#define IMB_AES128_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len) \
+
1856 ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
+
1857
+
1871#define IMB_AES256_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len) \
+
1872 ((_mgr)->aes256_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
+
1873
+
1874/* AES-GCM API's */
+
1875#define IMB_AES128_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1876 _aadl, _tag, _tagl) \
+
1877 ((_mgr)->gcm128_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1878 (_aad), (_aadl), (_tag), (_tagl)))
+
1879#define IMB_AES192_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1880 _aadl, _tag, _tagl) \
+
1881 ((_mgr)->gcm192_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1882 (_aad), (_aadl), (_tag), (_tagl)))
+
1883#define IMB_AES256_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1884 _aadl, _tag, _tagl) \
+
1885 ((_mgr)->gcm256_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1886 (_aad), (_aadl), (_tag), (_tagl)))
+
1887
+
1888#define IMB_AES128_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1889 _aadl, _tag, _tagl) \
+
1890 ((_mgr)->gcm128_dec((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1891 (_aad), (_aadl), (_tag), (_tagl)))
+
1892#define IMB_AES192_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, \
+
1893 _aad, _aadl, _tag, _tagl) \
+
1894 ((_mgr)->gcm192_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
1895 (_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
1896#define IMB_AES256_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, \
+
1897 _aad, _aadl, _tag, _tagl) \
+
1898 ((_mgr)->gcm256_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
1899 (_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
1900
+
1901#define IMB_AES128_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl) \
+
1902 ((_mgr)->gcm128_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
1903#define IMB_AES192_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl) \
+
1904 ((_mgr)->gcm192_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
1905#define IMB_AES256_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl) \
+
1906 ((_mgr)->gcm256_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
1907
+
1908#define IMB_AES128_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, \
+
1909 _aadl) \
+
1910 ((_mgr)->gcm128_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
1911 (_aad), (_aadl)))
+
1912#define IMB_AES192_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, \
+
1913 _aadl) \
+
1914 ((_mgr)->gcm192_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
1915 (_aad), (_aadl)))
+
1916#define IMB_AES256_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, \
+
1917 _aadl) \
+
1918 ((_mgr)->gcm256_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
1919 (_aad), (_aadl)))
+
1920
+
1921#define IMB_AES128_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1922 ((_mgr)->gcm128_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1923#define IMB_AES192_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1924 ((_mgr)->gcm192_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1925#define IMB_AES256_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1926 ((_mgr)->gcm256_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1927
+
1928#define IMB_AES128_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1929 ((_mgr)->gcm128_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1930#define IMB_AES192_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1931 ((_mgr)->gcm192_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1932#define IMB_AES256_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1933 ((_mgr)->gcm256_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1934
+
1935#define IMB_AES128_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1936 ((_mgr)->gcm128_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1937#define IMB_AES192_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1938 ((_mgr)->gcm192_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1939#define IMB_AES256_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1940 ((_mgr)->gcm256_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1941
+
1942#define IMB_AES128_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1943 ((_mgr)->gcm128_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1944#define IMB_AES192_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1945 ((_mgr)->gcm192_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1946#define IMB_AES256_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1947 ((_mgr)->gcm256_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1948
+
1949#define IMB_AES128_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl) \
+
1950 ((_mgr)->gmac128_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
1951#define IMB_AES192_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl) \
+
1952 ((_mgr)->gmac192_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
1953#define IMB_AES256_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl) \
+
1954 ((_mgr)->gmac256_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
1955
+
1956#define IMB_AES128_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len) \
+
1957 ((_mgr)->gmac128_update((_exp_key), (_ctx), (_src), (_len)))
+
1958#define IMB_AES192_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len) \
+
1959 ((_mgr)->gmac192_update((_exp_key), (_ctx), (_src), (_len)))
+
1960#define IMB_AES256_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len) \
+
1961 ((_mgr)->gmac256_update((_exp_key), (_ctx), (_src), (_len)))
+
1962
+
1963#define IMB_AES128_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1964 ((_mgr)->gmac128_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1965#define IMB_AES192_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1966 ((_mgr)->gmac192_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1967#define IMB_AES256_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1968 ((_mgr)->gmac256_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1969
+
1970#define IMB_AES128_GCM_PRECOMP(_mgr, _key) \
+
1971 ((_mgr)->gcm128_precomp((_key)))
+
1972#define IMB_AES192_GCM_PRECOMP(_mgr, _key) \
+
1973 ((_mgr)->gcm192_precomp((_key)))
+
1974#define IMB_AES256_GCM_PRECOMP(_mgr, _key) \
+
1975 ((_mgr)->gcm256_precomp((_key)))
+
1976
+
1977#define IMB_AES128_GCM_PRE(_mgr, _key, _exp_key) \
+
1978 ((_mgr)->gcm128_pre((_key), (_exp_key)))
+
1979#define IMB_AES192_GCM_PRE(_mgr, _key, _exp_key) \
+
1980 ((_mgr)->gcm192_pre((_key), (_exp_key)))
+
1981#define IMB_AES256_GCM_PRE(_mgr, _key, _exp_key) \
+
1982 ((_mgr)->gcm256_pre((_key), (_exp_key)))
+
1983
+
1984#define IMB_GHASH_PRE(_mgr, _key, _exp_key) \
+
1985 ((_mgr)->ghash_pre((_key), (_exp_key)))
+
1986#define IMB_GHASH(_mgr, _exp_key, _src, _len, _tag, _tagl) \
+
1987 ((_mgr)->ghash((_exp_key), (_src), (_len), (_tag), (_tagl)))
+
1988
+
1989/* Chacha20-Poly1305 direct API's */
+
1990#define IMB_CHACHA20_POLY1305_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl) \
+
1991 ((_mgr)->chacha20_poly1305_init((_key), (_ctx), (_iv), (_aad), \
+
1992 (_aadl)))
+
1993
+
1994#define IMB_CHACHA20_POLY1305_ENC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len) \
+
1995 ((_mgr)->chacha20_poly1305_enc_update((_key), (_ctx), (_dst), (_src), \
+
1996 (_len)))
+
1997#define IMB_CHACHA20_POLY1305_DEC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len) \
+
1998 ((_mgr)->chacha20_poly1305_dec_update((_key), (_ctx), (_dst), (_src), \
+
1999 (_len)))
+
2000
+
2001#define IMB_CHACHA20_POLY1305_ENC_FINALIZE(_mgr, _ctx, _tag, _tagl) \
+
2002 ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
2003
+
2004#define IMB_CHACHA20_POLY1305_DEC_FINALIZE(_mgr, _ctx, _tag, _tagl) \
+
2005 ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
2006
+
2007/* ZUC EEA3/EIA3 functions */
+
2008
+
2019#define IMB_ZUC_EEA3_1_BUFFER(_mgr, _key, _iv, _src, _dst, _len) \
+
2020 ((_mgr)->eea3_1_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
2021#define IMB_ZUC_EEA3_4_BUFFER(_mgr, _key, _iv, _src, _dst, _len) \
+
2022 ((_mgr)->eea3_4_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
2023#define IMB_ZUC_EEA3_N_BUFFER(_mgr, _key, _iv, _src, _dst, _len, _count) \
+
2024 ((_mgr)->eea3_n_buffer((_key), (_iv), (_src), (_dst), (_len), (_count)))
+
2025
+
2026
+
2037#define IMB_ZUC_EIA3_1_BUFFER(_mgr, _key, _iv, _src, _len, _tag) \
+
2038 ((_mgr)->eia3_1_buffer((_key), (_iv), (_src), (_len), (_tag)))
+
2039#define IMB_ZUC_EIA3_N_BUFFER(_mgr, _key, _iv, _src, _len, _tag, _count) \
+
2040 ((_mgr)->eia3_n_buffer((_key), (_iv), (_src), (_len), (_tag), (_count)))
+
2041
+
2042
+
2043/* KASUMI F8/F9 functions */
+
2044
+
2060#define IMB_KASUMI_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len) \
+
2061 ((_mgr)->f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
2062
+
2079#define IMB_KASUMI_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, _len, \
+
2080 _offset) \
+
2081 ((_mgr)->f8_1_buffer_bit((_exp_key), (_iv), (_src), (_dst), (_len), \
+
2082 (_offset)))
+
2083
+
2103#define IMB_KASUMI_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, _src1, _dst1, \
+
2104 _len1, _src2, _dst2, _len2) \
+
2105 ((_mgr)->f8_2_buffer((_exp_key), (_iv1), (_iv2), (_src1), (_dst1), \
+
2106 (_len1), (_src2), (_dst2), (_len2)))
+
2128#define IMB_KASUMI_F8_3_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _src1, _dst1, \
+
2129 _src2, _dst2, _src3, _dst3, _len) \
+
2130 ((_mgr)->f8_3_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_src1), \
+
2131 (_dst1), (_src2), (_dst2), (_src3), (_dst3), \
+
2132 (_len)))
+
2157#define IMB_KASUMI_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, \
+
2158 _src1, _dst1, _src2, _dst2, _src3, _dst3, \
+
2159 _src4, _dst4, _len) \
+
2160 ((_mgr)->f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_iv4), \
+
2161 (_src1), (_dst1), (_src2), (_dst2), \
+
2162 (_src3), (_dst3), (_src4), (_dst4), (_len)))
+
2178#define IMB_KASUMI_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count) \
+
2179 ((_mgr)->f8_n_buffer((_exp_key), (_iv), (_src), (_dst), (_len), \
+
2180 (_count)))
+
2196#define IMB_KASUMI_F9_1_BUFFER(_mgr, _exp_key, _src, _len, _tag) \
+
2197 ((_mgr)->f9_1_buffer((_exp_key), (_src), (_len), (_tag)))
+
2198
+
2213#define IMB_KASUMI_F9_1_BUFFER_USER(_mgr, _exp_key, _iv, _src, _len, _tag, \
+
2214 _dir) \
+
2215 ((_mgr)->f9_1_buffer_user((_exp_key), (_iv), (_src), (_len), \
+
2216 (_tag), (_dir)))
+
2217
+
2227#define IMB_KASUMI_INIT_F8_KEY_SCHED(_mgr, _key, _exp_key) \
+
2228 ((_mgr)->kasumi_init_f8_key_sched((_key), (_exp_key)))
+
2229
+
2239#define IMB_KASUMI_INIT_F9_KEY_SCHED(_mgr, _key, _exp_key) \
+
2240 ((_mgr)->kasumi_init_f9_key_sched((_key), (_exp_key)))
+
2241
+
2251#define IMB_KASUMI_KEY_SCHED_SIZE(_mgr)((_mgr)->kasumi_key_sched_size())
+
2252
+
2253
+
2254/* SNOW3G F8/F9 functions */
+
2255
+
2271#define IMB_SNOW3G_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, \
+
2272 _len, _offset) \
+
2273 ((_mgr)->snow3g_f8_1_buffer_bit((_exp_key), (_iv), (_src), \
+
2274 (_dst), (_len), (_offset)))
+
2275
+
2290#define IMB_SNOW3G_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len) \
+
2291 ((_mgr)->snow3g_f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
2292
+
2309#define IMB_SNOW3G_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, \
+
2310 _src1, _dst1, _len1, \
+
2311 _src2, _dst2, _len2) \
+
2312 ((_mgr)->snow3g_f8_2_buffer((_exp_key), (_iv1), (_iv2), \
+
2313 (_src1), (_dst1), (_len1), \
+
2314 (_src2), (_dst2), (_len2)))
+
2315
+
2341#define IMB_SNOW3G_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, \
+
2342 _src1, _dst1, _len1, \
+
2343 _src2, _dst2, _len2, \
+
2344 _src3, _dst3, _len3, \
+
2345 _src4, _dst4, _len4) \
+
2346 ((_mgr)->snow3g_f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
2347 (_iv4), (_src1), (_dst1), (_len1), \
+
2348 (_src2), (_dst2), (_len2), \
+
2349 (_src3), (_dst3), (_len3), \
+
2350 (_src4), (_dst4), (_len4)))
+
2351
+
2393#define IMB_SNOW3G_F8_8_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, \
+
2394 _iv5, _iv6, _iv7, _iv8, \
+
2395 _src1, _dst1, _len1, \
+
2396 _src2, _dst2, _len2, \
+
2397 _src3, _dst3, _len3, \
+
2398 _src4, _dst4, _len4, \
+
2399 _src5, _dst5, _len5, \
+
2400 _src6, _dst6, _len6, \
+
2401 _src7, _dst7, _len7, \
+
2402 _src8, _dst8, _len8) \
+
2403 ((_mgr)->snow3g_f8_8_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
2404 (_iv4), (_iv5), (_iv6), (_iv7), \
+
2405 (_iv8), (_src1), (_dst1), (_len1), \
+
2406 (_src2), (_dst2), (_len2), \
+
2407 (_src3), (_dst3), (_len3), \
+
2408 (_src4), (_dst4), (_len4), \
+
2409 (_src5), (_dst5), (_len5), \
+
2410 (_src6), (_dst6), (_len6), \
+
2411 (_src7), (_dst7), (_len7), \
+
2412 (_src8), (_dst8), (_len8)))
+
2426#define IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, _dst, _len) \
+
2427 ((_mgr)->snow3g_f8_8_buffer_multikey((_exp_key), (_iv), (_src), (_dst),\
+
2428 (_len)))
+
2429
+
2444#define IMB_SNOW3G_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count) \
+
2445 ((_mgr)->snow3g_f8_n_buffer((_exp_key), (_iv), (_src), \
+
2446 (_dst), (_len), (_count)))
+
2447
+
2462#define IMB_SNOW3G_F8_N_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, \
+
2463 _dst, _len, _count) \
+
2464 ((_mgr)->snow3g_f8_n_buffer_multikey((_exp_key), (_iv), (_src), \
+
2465 (_dst), (_len), (_count)))
+
2466
+
2482#define IMB_SNOW3G_F9_1_BUFFER(_mgr, _exp_key, _iv, _src, _len, _tag) \
+
2483 ((_mgr)->snow3g_f9_1_buffer((_exp_key), (_iv), (_src), (_len), (_tag)))
+
2484
+
2495#define IMB_SNOW3G_INIT_KEY_SCHED(_mgr, _key, _exp_key) \
+
2496 ((_mgr)->snow3g_init_key_sched((_key), (_exp_key)))
+
2497
+
2507#define IMB_SNOW3G_KEY_SCHED_SIZE(_mgr)((_mgr)->snow3g_key_sched_size())
+
2508
+
2512#define IMB_HEC_32(_mgr, _src)((_mgr)->hec_32(_src))
+
2513#define IMB_HEC_64(_mgr, _src)((_mgr)->hec_64(_src))
+
2514
+
2518#define IMB_CRC32_ETHERNET_FCS(_mgr, _src, _len) \
+
2519 (_mgr)->crc32_ethernet_fcs(_src, _len)
+
2520
+
2524#define IMB_CRC16_X25(_mgr, _src, _len) \
+
2525 (_mgr)->crc16_x25(_src, _len)
+
2526
+
2530#define IMB_CRC32_SCTP(_mgr, _src, _len) \
+
2531 (_mgr)->crc32_sctp(_src, _len)
+
2532
+
2536#define IMB_CRC24_LTE_A(_mgr, _src, _len) \
+
2537 (_mgr)->crc24_lte_a(_src, _len)
+
2538
+
2542#define IMB_CRC24_LTE_B(_mgr, _src, _len) \
+
2543 (_mgr)->crc24_lte_b(_src, _len)
+
2544
+
2548#define IMB_CRC16_FP_DATA(_mgr, _src, _len) \
+
2549 (_mgr)->crc16_fp_data(_src, _len)
+
2550
+
2554#define IMB_CRC11_FP_HEADER(_mgr, _src, _len) \
+
2555 (_mgr)->crc11_fp_header(_src, _len)
+
2556
+
2560#define IMB_CRC7_FP_HEADER(_mgr, _src, _len) \
+
2561 (_mgr)->crc7_fp_header(_src, _len)
+
2562
+
2566#define IMB_CRC10_IUUP_DATA(_mgr, _src, _len) \
+
2567 (_mgr)->crc10_iuup_data(_src, _len)
+
2568
+
2572#define IMB_CRC6_IUUP_HEADER(_mgr, _src, _len) \
+
2573 (_mgr)->crc6_iuup_header(_src, _len)
+
2574
+
2578#define IMB_CRC32_WIMAX_OFDMA_DATA(_mgr, _src, _len) \
+
2579 (_mgr)->crc32_wimax_ofdma_data(_src, _len)
+
2580
+
2584#define IMB_CRC8_WIMAX_OFDMA_HCS(_mgr, _src, _len) \
+
2585 (_mgr)->crc8_wimax_ofdma_hcs(_src, _len)
+
2586
+
2587/* Auxiliary functions */
+
2588
+
2601IMB_DLL_EXPORT int
+
2602des_key_schedule(uint64_t *ks, const void *key);
+
2603
+
2614IMB_DLL_EXPORT void
+
2615imb_hmac_ipad_opad(struct IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type,
+
2616 const void *pkey, const size_t key_len,
+
2617 void *ipad_hash, void *opad_hash);
+
2618
+
2631IMB_DLL_EXPORT void
+
2632des_cfb_one(void *out, const void *in, const uint64_t *iv,
+
2633 const uint64_t *ks, const int len);
+
2634
+
2641IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest);
+
2645IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest);
+
2649IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest);
+
2653IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest);
+
2654
+
2655
+
2663IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys,
+
2664 void *dec_exp_keys);
+
2668IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys,
+
2669 void *dec_exp_keys);
+
2673IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys,
+
2674 void *dec_exp_keys);
+
2678IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys,
+
2679 void *dec_exp_keys);
+
2680
+
2688IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys,
+
2689 void *dec_exp_keys);
+
2693IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys,
+
2694 void *dec_exp_keys);
+
2698IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys,
+
2699 void *dec_exp_keys);
+
2703IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys,
+
2704 void *dec_exp_keys);
+
2705
+
2713IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys,
+
2714 void *dec_exp_keys);
+
2718IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys,
+
2719 void *dec_exp_keys);
+
2723IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys,
+
2724 void *dec_exp_keys);
+
2728IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys,
+
2729 void *dec_exp_keys);
+
2730
+
2737IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key,
+
2738 void *enc_exp_keys);
+
2742IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key,
+
2743 void *enc_exp_keys);
+
2747IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key,
+
2748 void *enc_exp_keys);
+
2752IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key,
+
2753 void *enc_exp_keys);
+
2754
+
2761IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key,
+
2762 void *enc_exp_keys);
+
2766IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key,
+
2767 void *enc_exp_keys);
+
2771IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key,
+
2772 void *enc_exp_keys);
+
2776IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key,
+
2777 void *enc_exp_keys);
+
2778
+
2785IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key,
+
2786 void *enc_exp_keys);
+
2790IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key,
+
2791 void *enc_exp_keys);
+
2795IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key,
+
2796 void *enc_exp_keys);
+
2800IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key,
+
2801 void *enc_exp_keys);
+
2802
+
2811IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp,
+
2812 void *k2, void *k3);
+
2816IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp,
+
2817 void *k2, void *k3);
+
2821IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp,
+
2822 void *k2, void *k3);
+
2826IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp,
+
2827 void *k2, void *k3);
+
2828
+
2836IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1,
+
2837 void *key2);
+
2841IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1,
+
2842 void *key2);
+
2846IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1,
+
2847 void *key2);
+
2851IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1,
+
2852 void *key2);
+
2853
+
2854/*
+
2855 * Direct GCM API.
+
2856 * Note that GCM is also available through job API.
+
2857 */
+
2858
+
2877IMB_DLL_EXPORT void
+
2878aes_gcm_enc_128_sse(const struct gcm_key_data *key_data,
+
2879 struct gcm_context_data *context_data,
+
2880 uint8_t *out, uint8_t const *in, uint64_t len,
+
2881 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
2882 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2886IMB_DLL_EXPORT void
+
2887aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data,
+
2888 struct gcm_context_data *context_data,
+
2889 uint8_t *out, uint8_t const *in, uint64_t len,
+
2890 const uint8_t *iv,
+
2891 uint8_t const *aad, uint64_t aad_len,
+
2892 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2896IMB_DLL_EXPORT void
+
2897aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data,
+
2898 struct gcm_context_data *context_data,
+
2899 uint8_t *out, uint8_t const *in, uint64_t len,
+
2900 const uint8_t *iv,
+
2901 uint8_t const *aad, uint64_t aad_len,
+
2902 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2921IMB_DLL_EXPORT void
+
2922aes_gcm_enc_192_sse(const struct gcm_key_data *key_data,
+
2923 struct gcm_context_data *context_data,
+
2924 uint8_t *out, uint8_t const *in, uint64_t len,
+
2925 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
2926 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2930IMB_DLL_EXPORT void
+
2931aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data,
+
2932 struct gcm_context_data *context_data,
+
2933 uint8_t *out, uint8_t const *in, uint64_t len,
+
2934 const uint8_t *iv,
+
2935 uint8_t const *aad, uint64_t aad_len,
+
2936 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2940IMB_DLL_EXPORT void
+
2941aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data,
+
2942 struct gcm_context_data *context_data,
+
2943 uint8_t *out, uint8_t const *in, uint64_t len,
+
2944 const uint8_t *iv,
+
2945 uint8_t const *aad, uint64_t aad_len,
+
2946 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2947
+
2966IMB_DLL_EXPORT void
+
2967aes_gcm_enc_256_sse(const struct gcm_key_data *key_data,
+
2968 struct gcm_context_data *context_data,
+
2969 uint8_t *out, uint8_t const *in, uint64_t len,
+
2970 const uint8_t *iv,
+
2971 uint8_t const *aad, uint64_t aad_len,
+
2972 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2976IMB_DLL_EXPORT void
+
2977aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data,
+
2978 struct gcm_context_data *context_data,
+
2979 uint8_t *out, uint8_t const *in, uint64_t len,
+
2980 const uint8_t *iv,
+
2981 uint8_t const *aad, uint64_t aad_len,
+
2982 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2986IMB_DLL_EXPORT void
+
2987aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data,
+
2988 struct gcm_context_data *context_data,
+
2989 uint8_t *out, uint8_t const *in, uint64_t len,
+
2990 const uint8_t *iv,
+
2991 uint8_t const *aad, uint64_t aad_len,
+
2992 uint8_t *auth_tag, uint64_t auth_tag_len);
+
2993
+
3012IMB_DLL_EXPORT void
+
3013aes_gcm_dec_128_sse(const struct gcm_key_data *key_data,
+
3014 struct gcm_context_data *context_data,
+
3015 uint8_t *out, uint8_t const *in, uint64_t len,
+
3016 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3017 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3021IMB_DLL_EXPORT void
+
3022aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data,
+
3023 struct gcm_context_data *context_data,
+
3024 uint8_t *out, uint8_t const *in, uint64_t len,
+
3025 const uint8_t *iv,
+
3026 uint8_t const *aad, uint64_t aad_len,
+
3027 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3031IMB_DLL_EXPORT void
+
3032aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data,
+
3033 struct gcm_context_data *context_data,
+
3034 uint8_t *out, uint8_t const *in, uint64_t len,
+
3035 const uint8_t *iv,
+
3036 uint8_t const *aad, uint64_t aad_len,
+
3037 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3038
+
3057IMB_DLL_EXPORT void
+
3058aes_gcm_dec_192_sse(const struct gcm_key_data *key_data,
+
3059 struct gcm_context_data *context_data,
+
3060 uint8_t *out, uint8_t const *in, uint64_t len,
+
3061 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3062 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3066IMB_DLL_EXPORT void
+
3067aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data,
+
3068 struct gcm_context_data *context_data,
+
3069 uint8_t *out, uint8_t const *in, uint64_t len,
+
3070 const uint8_t *iv,
+
3071 uint8_t const *aad, uint64_t aad_len,
+
3072 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3076IMB_DLL_EXPORT void
+
3077aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data,
+
3078 struct gcm_context_data *context_data,
+
3079 uint8_t *out, uint8_t const *in, uint64_t len,
+
3080 const uint8_t *iv,
+
3081 uint8_t const *aad, uint64_t aad_len,
+
3082 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3083
+
3102IMB_DLL_EXPORT void
+
3103aes_gcm_dec_256_sse(const struct gcm_key_data *key_data,
+
3104 struct gcm_context_data *context_data,
+
3105 uint8_t *out, uint8_t const *in, uint64_t len,
+
3106 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3107 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3111IMB_DLL_EXPORT void
+
3112aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data,
+
3113 struct gcm_context_data *context_data,
+
3114 uint8_t *out, uint8_t const *in, uint64_t len,
+
3115 const uint8_t *iv,
+
3116 uint8_t const *aad, uint64_t aad_len,
+
3117 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3121IMB_DLL_EXPORT void
+
3122aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data,
+
3123 struct gcm_context_data *context_data,
+
3124 uint8_t *out, uint8_t const *in, uint64_t len,
+
3125 const uint8_t *iv,
+
3126 uint8_t const *aad, uint64_t aad_len,
+
3127 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3128
+
3141IMB_DLL_EXPORT void
+
3142aes_gcm_init_128_sse(const struct gcm_key_data *key_data,
+
3143 struct gcm_context_data *context_data,
+
3144 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+
3148IMB_DLL_EXPORT void
+
3149aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data,
+
3150 struct gcm_context_data *context_data,
+
3151 const uint8_t *iv,
+
3152 uint8_t const *aad, uint64_t aad_len);
+
3156IMB_DLL_EXPORT void
+
3157aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data,
+
3158 struct gcm_context_data *context_data,
+
3159 const uint8_t *iv,
+
3160 uint8_t const *aad, uint64_t aad_len);
+
3173IMB_DLL_EXPORT void
+
3174aes_gcm_init_192_sse(const struct gcm_key_data *key_data,
+
3175 struct gcm_context_data *context_data,
+
3176 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+
3180IMB_DLL_EXPORT void
+
3181aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data,
+
3182 struct gcm_context_data *context_data,
+
3183 const uint8_t *iv,
+
3184 uint8_t const *aad, uint64_t aad_len);
+
3188IMB_DLL_EXPORT void
+
3189aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data,
+
3190 struct gcm_context_data *context_data,
+
3191 const uint8_t *iv,
+
3192 uint8_t const *aad, uint64_t aad_len);
+
3205IMB_DLL_EXPORT void
+
3206aes_gcm_init_256_sse(const struct gcm_key_data *key_data,
+
3207 struct gcm_context_data *context_data,
+
3208 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+
3212IMB_DLL_EXPORT void
+
3213aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data,
+
3214 struct gcm_context_data *context_data,
+
3215 const uint8_t *iv,
+
3216 uint8_t const *aad, uint64_t aad_len);
+
3220IMB_DLL_EXPORT void
+
3221aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data,
+
3222 struct gcm_context_data *context_data,
+
3223 const uint8_t *iv,
+
3224 uint8_t const *aad, uint64_t aad_len);
+
3225
+
3235IMB_DLL_EXPORT void
+
3236aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data,
+
3237 struct gcm_context_data *context_data,
+
3238 uint8_t *out, const uint8_t *in, uint64_t len);
+
3242IMB_DLL_EXPORT void
+
3243aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data,
+
3244 struct gcm_context_data *context_data,
+
3245 uint8_t *out, const uint8_t *in, uint64_t len);
+
3249IMB_DLL_EXPORT void
+
3250aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data,
+
3251 struct gcm_context_data *context_data,
+
3252 uint8_t *out, const uint8_t *in, uint64_t len);
+
3253
+
3263IMB_DLL_EXPORT void
+
3264aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data,
+
3265 struct gcm_context_data *context_data,
+
3266 uint8_t *out, const uint8_t *in, uint64_t len);
+
3270IMB_DLL_EXPORT void
+
3271aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data,
+
3272 struct gcm_context_data *context_data,
+
3273 uint8_t *out, const uint8_t *in, uint64_t len);
+
3277IMB_DLL_EXPORT void
+
3278aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data,
+
3279 struct gcm_context_data *context_data,
+
3280 uint8_t *out, const uint8_t *in, uint64_t len);
+
3281
+
3291IMB_DLL_EXPORT void
+
3292aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data,
+
3293 struct gcm_context_data *context_data,
+
3294 uint8_t *out, const uint8_t *in, uint64_t len);
+
3298IMB_DLL_EXPORT void
+
3299aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data,
+
3300 struct gcm_context_data *context_data,
+
3301 uint8_t *out, const uint8_t *in, uint64_t len);
+
3305IMB_DLL_EXPORT void
+
3306aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data,
+
3307 struct gcm_context_data *context_data,
+
3308 uint8_t *out, const uint8_t *in, uint64_t len);
+
3309
+
3319IMB_DLL_EXPORT void
+
3320aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data,
+
3321 struct gcm_context_data *context_data,
+
3322 uint8_t *out, const uint8_t *in, uint64_t len);
+
3326IMB_DLL_EXPORT void
+
3327aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data,
+
3328 struct gcm_context_data *context_data,
+
3329 uint8_t *out, const uint8_t *in, uint64_t len);
+
3333IMB_DLL_EXPORT void
+
3334aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data,
+
3335 struct gcm_context_data *context_data,
+
3336 uint8_t *out, const uint8_t *in, uint64_t len);
+
3337
+
3347IMB_DLL_EXPORT void
+
3348aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data,
+
3349 struct gcm_context_data *context_data,
+
3350 uint8_t *out, const uint8_t *in, uint64_t len);
+
3354IMB_DLL_EXPORT void
+
3355aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data,
+
3356 struct gcm_context_data *context_data,
+
3357 uint8_t *out, const uint8_t *in, uint64_t len);
+
3361IMB_DLL_EXPORT void
+
3362aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data,
+
3363 struct gcm_context_data *context_data,
+
3364 uint8_t *out, const uint8_t *in, uint64_t len);
+
3365
+
3375IMB_DLL_EXPORT void
+
3376aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data,
+
3377 struct gcm_context_data *context_data,
+
3378 uint8_t *out, const uint8_t *in, uint64_t len);
+
3382IMB_DLL_EXPORT void
+
3383aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data,
+
3384 struct gcm_context_data *context_data,
+
3385 uint8_t *out, const uint8_t *in, uint64_t len);
+
3389IMB_DLL_EXPORT void
+
3390aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data,
+
3391 struct gcm_context_data *context_data,
+
3392 uint8_t *out, const uint8_t *in, uint64_t len);
+
3393
+
3404IMB_DLL_EXPORT void
+
3405aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data,
+
3406 struct gcm_context_data *context_data,
+
3407 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3411IMB_DLL_EXPORT void
+
3412aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3413 struct gcm_context_data *context_data,
+
3414 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3418IMB_DLL_EXPORT void
+
3419aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3420 struct gcm_context_data *context_data,
+
3421 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3422
+
3433IMB_DLL_EXPORT void
+
3434aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data,
+
3435 struct gcm_context_data *context_data,
+
3436 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3440IMB_DLL_EXPORT void
+
3441aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3442 struct gcm_context_data *context_data,
+
3443 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3447IMB_DLL_EXPORT void
+
3448aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3449 struct gcm_context_data *context_data,
+
3450 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3451
+
3462IMB_DLL_EXPORT void
+
3463aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data,
+
3464 struct gcm_context_data *context_data,
+
3465 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3469IMB_DLL_EXPORT void
+
3470aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3471 struct gcm_context_data *context_data,
+
3472 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3476IMB_DLL_EXPORT void
+
3477aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3478 struct gcm_context_data *context_data,
+
3479 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3480
+
3491IMB_DLL_EXPORT void
+
3492aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data,
+
3493 struct gcm_context_data *context_data,
+
3494 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3498IMB_DLL_EXPORT void
+
3499aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3500 struct gcm_context_data *context_data,
+
3501 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3505IMB_DLL_EXPORT void
+
3506aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3507 struct gcm_context_data *context_data,
+
3508 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3509
+
3520IMB_DLL_EXPORT void
+
3521aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data,
+
3522 struct gcm_context_data *context_data,
+
3523 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3527IMB_DLL_EXPORT void
+
3528aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3529 struct gcm_context_data *context_data,
+
3530 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3534IMB_DLL_EXPORT void
+
3535aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3536 struct gcm_context_data *context_data,
+
3537 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3538
+
3549IMB_DLL_EXPORT void
+
3550aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data,
+
3551 struct gcm_context_data *context_data,
+
3552 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3556IMB_DLL_EXPORT void
+
3557aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3558 struct gcm_context_data *context_data,
+
3559 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3563IMB_DLL_EXPORT void
+
3564aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3565 struct gcm_context_data *context_data,
+
3566 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3567
+
3576IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data);
+
3577
+
3581IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data);
+
3582
+
3586IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data);
+
3587
+
3596IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data);
+
3597
+
3601IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data);
+
3602
+
3606IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data);
+
3607
+
3616IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data);
+
3617
+
3621IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data);
+
3622
+
3626IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data);
+
3627
+
3637IMB_DLL_EXPORT void aes_gcm_pre_128_sse(const void *key,
+
3638 struct gcm_key_data *key_data);
+
3642IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2(const void *key,
+
3643 struct gcm_key_data *key_data);
+
3647IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4(const void *key,
+
3648 struct gcm_key_data *key_data);
+
3658IMB_DLL_EXPORT void aes_gcm_pre_192_sse(const void *key,
+
3659 struct gcm_key_data *key_data);
+
3663IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2(const void *key,
+
3664 struct gcm_key_data *key_data);
+
3668IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4(const void *key,
+
3669 struct gcm_key_data *key_data);
+
3679IMB_DLL_EXPORT void aes_gcm_pre_256_sse(const void *key,
+
3680 struct gcm_key_data *key_data);
+
3684IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2(const void *key,
+
3685 struct gcm_key_data *key_data);
+
3689IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key,
+
3690 struct gcm_key_data *key_data);
+
3691
+
3704IMB_DLL_EXPORT int zuc_eea3_iv_gen(const uint32_t count,
+
3705 const uint8_t bearer,
+
3706 const uint8_t dir,
+
3707 void *iv_ptr);
+
3720IMB_DLL_EXPORT int zuc_eia3_iv_gen(const uint32_t count,
+
3721 const uint8_t bearer,
+
3722 const uint8_t dir,
+
3723 void *iv_ptr);
+
3724
+
3737IMB_DLL_EXPORT int kasumi_f8_iv_gen(const uint32_t count,
+
3738 const uint8_t bearer,
+
3739 const uint8_t dir,
+
3740 void *iv_ptr);
+
3752IMB_DLL_EXPORT int kasumi_f9_iv_gen(const uint32_t count,
+
3753 const uint32_t fresh,
+
3754 void *iv_ptr);
+
3755
+
3771IMB_DLL_EXPORT int snow3g_f8_iv_gen(const uint32_t count,
+
3772 const uint8_t bearer,
+
3773 const uint8_t dir,
+
3774 void *iv_ptr);
+
3790IMB_DLL_EXPORT int snow3g_f9_iv_gen(const uint32_t count,
+
3791 const uint32_t fresh,
+
3792 const uint8_t dir,
+
3793 void *iv_ptr);
+
3800IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size);
+
3801
+
3823IMB_DLL_EXPORT void
+
3824imb_quic_aes_gcm(IMB_MGR *state,
+
3825 const struct gcm_key_data *key_data,
+
3826 const IMB_KEY_SIZE_BYTES key_size,
+
3827 const IMB_CIPHER_DIRECTION cipher_dir,
+
3828 void *dst_ptr_array[],
+
3829 const void * const src_ptr_array[],
+
3830 const uint64_t len_array[],
+
3831 const void * const iv_ptr_array[],
+
3832 const void * const aad_ptr_array[],
+
3833 const uint64_t aad_len,
+
3834 void *tag_ptr_array[],
+
3835 const uint64_t tag_len,
+
3836 const uint64_t num_packets);
+
3837
+
3852IMB_DLL_EXPORT void
+
3853imb_quic_hp_aes_ecb(IMB_MGR *state,
+
3854 const void *exp_key_data,
+
3855 void *dst_ptr_array[],
+
3856 const void * const src_ptr_array[],
+
3857 const uint64_t num_packets,
+
3858 const IMB_KEY_SIZE_BYTES key_size);
+
3859
+
3899IMB_DLL_EXPORT uint32_t imb_set_session(IMB_MGR *state, IMB_JOB *job);
+
3900
+
3901#ifdef __cplusplus
+
3902}
+
3903#endif
+
3904
+
3905#endif /* IMB_IPSEC_MB_H */
+
aes_gcm_enc_128_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-128 encryption message.
+
get_next_job_sse
IMB_DLL_EXPORT IMB_JOB * get_next_job_sse(IMB_MGR *state)
Get next available job.
+
KASUMI_KEY_SCHEDULE_SIZE
#define KASUMI_KEY_SCHEDULE_SIZE
Definition intel-ipsec-mb.h:558
+
aes_gcm_pre_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-256 key data.
+
aes_gcm_dec_128_finalize_sse
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-128 encryption message.
+
aes_gcm_init_256_sse
IMB_DLL_EXPORT void aes_gcm_init_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
+
zuc_eea3_1_buffer_t
void(* zuc_eea3_1_buffer_t)(const void *, const void *, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:770
+
aes_keyexp_192_enc_sse
IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key, void *enc_exp_keys)
+
aes_gcm_dec_256_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-256 encryption message.
+
IMB_GCM_ENC_KEY_LEN
#define IMB_GCM_ENC_KEY_LEN
Definition intel-ipsec-mb.h:628
+
aes_gcm_dec_192_finalize_sse
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-192 encryption message.
+
aes_gcm_enc_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Encryption.
+
snow3g_f8_8_buffer_multikey_t
void(* snow3g_f8_8_buffer_multikey_t)(const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[])
Definition intel-ipsec-mb.h:879
+
aes_gcm_pre_t
void(* aes_gcm_pre_t)(const void *, struct gcm_key_data *)
Definition intel-ipsec-mb.h:747
+
ghash_t
void(* ghash_t)(const struct gcm_key_data *, const void *, const uint64_t, void *, const uint64_t)
Definition intel-ipsec-mb.h:767
+
aes_xcbc_expand_key_avx512
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp, void *k2, void *k3)
+
aes_gcm_dec_256_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-256 encryption message.
+
aes_gcm_enc_192_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-192 encryption message.
+
aes_gcm_precomp_256_sse
IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data)
Precomputation of AES-GCM-256 HashKey constants.
+
aes_gcm_pre_256_sse
IMB_DLL_EXPORT void aes_gcm_pre_256_sse(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-256 key data.
+
chacha_poly_finalize_t
void(* chacha_poly_finalize_t)(struct chacha20_poly1305_context_data *, void *, const uint64_t)
Definition intel-ipsec-mb.h:765
+
aes_keyexp_192_avx512
IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
IMB_KEY_SIZE_BYTES
IMB_KEY_SIZE_BYTES
Definition intel-ipsec-mb.h:348
+
IMB_KEY_192_BYTES
@ IMB_KEY_192_BYTES
Definition intel-ipsec-mb.h:351
+
IMB_KEY_64_BYTES
@ IMB_KEY_64_BYTES
Definition intel-ipsec-mb.h:349
+
IMB_KEY_256_BYTES
@ IMB_KEY_256_BYTES
Definition intel-ipsec-mb.h:352
+
IMB_KEY_128_BYTES
@ IMB_KEY_128_BYTES
Definition intel-ipsec-mb.h:350
+
aes_keyexp_256_enc_avx
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key, void *enc_exp_keys)
+
aes_gcm_dec_128_update_sse
IMB_DLL_EXPORT void aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-128 encryption message.
+
free_mb_mgr
IMB_DLL_EXPORT void free_mb_mgr(IMB_MGR *ptr)
Frees memory allocated previously by alloc_mb_mgr()
+
get_completed_job_sse
IMB_DLL_EXPORT IMB_JOB * get_completed_job_sse(IMB_MGR *state)
Get next completed job.
+
aes_keyexp_128_enc_sse
IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key, void *enc_exp_keys)
+
snow3g_f8_1_buffer_t
void(* snow3g_f8_1_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:844
+
aes_gcm_precomp_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data)
Precomputation of AES-GCM-256 HashKey constants.
+
init_mb_mgr_sse
IMB_DLL_EXPORT void init_mb_mgr_sse(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
init_mb_mgr_auto
IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch)
Automatically initialize most performant Multi-buffer manager based on CPU features.
+
snow3g_f8_iv_gen
IMB_DLL_EXPORT int snow3g_f8_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of SNOW3G F8 Initialization Vector.
+
aes_gcm_pre_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-128 key data.
+
zuc_eea3_n_buffer_t
void(* zuc_eea3_n_buffer_t)(const void *const *, const void *const *, const void *const *, void **, const uint32_t *, const uint32_t)
Definition intel-ipsec-mb.h:777
+
aes_gmac_init_t
void(* aes_gmac_init_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
Definition intel-ipsec-mb.h:749
+
aes_keyexp_128_sse
IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
get_next_job_avx
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx(IMB_MGR *state)
Get next available job.
+
aes_gcm_enc_256_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-256 encryption message.
+
submit_job_nocheck_sse
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_sse(IMB_MGR *state)
Submit job for processing without validating.
+
aes_keyexp_256_avx512
IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
kasumi_f8_iv_gen
IMB_DLL_EXPORT int kasumi_f8_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of KASUMI F8 Initialization Vector.
+
imb_set_pointers_mb_mgr
IMB_DLL_EXPORT IMB_MGR * imb_set_pointers_mb_mgr(void *ptr, const uint64_t flags, const unsigned reset_mgr)
Initializes IMB_MGR pointers to out-of-order managers with use of externally allocated memory.
+
aes_keyexp_192_avx
IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
IMB_CIPHER_MODE
IMB_CIPHER_MODE
Definition intel-ipsec-mb.h:261
+
IMB_CIPHER_DES
@ IMB_CIPHER_DES
Definition intel-ipsec-mb.h:268
+
IMB_CIPHER_NULL
@ IMB_CIPHER_NULL
Definition intel-ipsec-mb.h:264
+
IMB_CIPHER_CBC
@ IMB_CIPHER_CBC
Definition intel-ipsec-mb.h:262
+
IMB_CIPHER_GCM
@ IMB_CIPHER_GCM
Definition intel-ipsec-mb.h:266
+
IMB_CIPHER_ECB
@ IMB_CIPHER_ECB
Definition intel-ipsec-mb.h:273
+
IMB_CIPHER_ZUC_EEA3
@ IMB_CIPHER_ZUC_EEA3
Definition intel-ipsec-mb.h:275
+
IMB_CIPHER_CHACHA20
@ IMB_CIPHER_CHACHA20
Definition intel-ipsec-mb.h:279
+
IMB_CIPHER_KASUMI_UEA1_BITLEN
@ IMB_CIPHER_KASUMI_UEA1_BITLEN
Definition intel-ipsec-mb.h:277
+
IMB_CIPHER_CHACHA20_POLY1305_SGL
@ IMB_CIPHER_CHACHA20_POLY1305_SGL
Definition intel-ipsec-mb.h:281
+
IMB_CIPHER_CUSTOM
@ IMB_CIPHER_CUSTOM
Definition intel-ipsec-mb.h:267
+
IMB_CIPHER_GCM_SGL
@ IMB_CIPHER_GCM_SGL
Definition intel-ipsec-mb.h:284
+
IMB_CIPHER_CNTR
@ IMB_CIPHER_CNTR
Definition intel-ipsec-mb.h:263
+
IMB_CIPHER_DES3
@ IMB_CIPHER_DES3
Definition intel-ipsec-mb.h:271
+
IMB_CIPHER_CNTR_BITLEN
@ IMB_CIPHER_CNTR_BITLEN
Definition intel-ipsec-mb.h:274
+
IMB_CIPHER_CBCS_1_9
@ IMB_CIPHER_CBCS_1_9
Definition intel-ipsec-mb.h:278
+
IMB_CIPHER_NUM
@ IMB_CIPHER_NUM
Definition intel-ipsec-mb.h:285
+
IMB_CIPHER_CCM
@ IMB_CIPHER_CCM
Definition intel-ipsec-mb.h:270
+
IMB_CIPHER_SNOW3G_UEA2_BITLEN
@ IMB_CIPHER_SNOW3G_UEA2_BITLEN
Definition intel-ipsec-mb.h:276
+
IMB_CIPHER_PON_AES_CNTR
@ IMB_CIPHER_PON_AES_CNTR
Definition intel-ipsec-mb.h:272
+
IMB_CIPHER_CHACHA20_POLY1305
@ IMB_CIPHER_CHACHA20_POLY1305
Definition intel-ipsec-mb.h:280
+
IMB_CIPHER_SNOW_V_AEAD
@ IMB_CIPHER_SNOW_V_AEAD
Definition intel-ipsec-mb.h:283
+
IMB_CIPHER_DOCSIS_SEC_BPI
@ IMB_CIPHER_DOCSIS_SEC_BPI
Definition intel-ipsec-mb.h:265
+
IMB_CIPHER_DOCSIS_DES
@ IMB_CIPHER_DOCSIS_DES
Definition intel-ipsec-mb.h:269
+
IMB_CIPHER_SNOW_V
@ IMB_CIPHER_SNOW_V
Definition intel-ipsec-mb.h:282
+
aes_gcm_pre_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-128 key data.
+
aes_gcm_init_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
+
snow3g_f8_n_buffer_multikey_t
void(* snow3g_f8_n_buffer_multikey_t)(const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
Definition intel-ipsec-mb.h:890
+
aes_gcm_enc_128_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-128 encryption message.
+
get_next_job_avx512
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx512(IMB_MGR *state)
Get next available job.
+
submit_job_t
IMB_JOB *(* submit_job_t)(struct IMB_MGR *)
Definition intel-ipsec-mb.h:697
+
chacha_poly_enc_dec_update_t
void(* chacha_poly_enc_dec_update_t)(const void *, struct chacha20_poly1305_context_data *, void *, const void *, const uint64_t)
Definition intel-ipsec-mb.h:762
+
aes_gcm_dec_128_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-128 encryption message.
+
imb_get_version_str
IMB_DLL_EXPORT const char * imb_get_version_str(void)
Get library version in string format.
+
aes_gcm_enc_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Encryption.
+
aes_keyexp_192_enc_avx2
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key, void *enc_exp_keys)
+
aes_gcm_enc_256_sse
IMB_DLL_EXPORT void aes_gcm_enc_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Encryption.
+
aes_gcm_precomp_128_sse
IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data)
Precomputation of AES-GCM-128 HashKey constants.
+
aes_gcm_enc_192_update_sse
IMB_DLL_EXPORT void aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-192 encryption message.
+
zuc_eea3_iv_gen
IMB_DLL_EXPORT int zuc_eea3_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of ZUC-EEA3 Initialization Vector.
+
get_next_job_avx2
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx2(IMB_MGR *state)
Get next available job.
+
IMB_CHAIN_ORDER
IMB_CHAIN_ORDER
Definition intel-ipsec-mb.h:343
+
IMB_ORDER_HASH_CIPHER
@ IMB_ORDER_HASH_CIPHER
Definition intel-ipsec-mb.h:345
+
IMB_ORDER_CIPHER_HASH
@ IMB_ORDER_CIPHER_HASH
Definition intel-ipsec-mb.h:344
+
chacha_poly_init_t
void(* chacha_poly_init_t)(const void *, struct chacha20_poly1305_context_data *, const void *, const void *, const uint64_t)
Definition intel-ipsec-mb.h:759
+
aes_cmac_subkey_gen_sse
IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1, void *key2)
+
snow3g_f8_2_buffer_t
void(* snow3g_f8_2_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:852
+
submit_hash_burst_t
uint32_t(* submit_hash_burst_t)(struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_HASH_ALG hash)
Definition intel-ipsec-mb.h:710
+
aes_gcm_enc_192_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-192 encryption message.
+
aes_keyexp_128_avx2
IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_keyexp_256_sse
IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_gcm_enc_192_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-192 encryption message.
+
kasumi_f8_n_buffer_t
void(* kasumi_f8_n_buffer_t)(const kasumi_key_sched_t *, const uint64_t *, const void *const *, void **, const uint32_t *, const uint32_t)
Definition intel-ipsec-mb.h:818
+
queue_size_avx512
IMB_DLL_EXPORT uint32_t queue_size_avx512(IMB_MGR *state)
Get number of jobs queued to be processed.
+
aes_gcm_enc_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Encryption.
+
aes_keyexp_192_enc_avx
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key, void *enc_exp_keys)
+
kasumi_f8_3_buffer_t
void(* kasumi_f8_3_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:803
+
aes_gcm_enc_192_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-192 encryption message.
+
crc32_fn_t
uint32_t(* crc32_fn_t)(const void *, const uint64_t)
Definition intel-ipsec-mb.h:908
+
aes_gcm_dec_192_update_sse
IMB_DLL_EXPORT void aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-192 encryption message.
+
IMB_GCM_BLOCK_LEN
#define IMB_GCM_BLOCK_LEN
Definition intel-ipsec-mb.h:570
+
get_next_job_t
IMB_JOB *(* get_next_job_t)(struct IMB_MGR *)
Definition intel-ipsec-mb.h:696
+
submit_job_nocheck_avx512
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx512(IMB_MGR *state)
Submit job for processing without validating.
+
aes_keyexp_256_avx
IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_gcm_init_192_sse
IMB_DLL_EXPORT void aes_gcm_init_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
+
aes_gcm_dec_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Decryption.
+
aes_gcm_pre_128_sse
IMB_DLL_EXPORT void aes_gcm_pre_128_sse(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-128 key data.
+
aes_gcm_enc_192_finalize_sse
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-192 encryption message.
+
des_keysched_t
int(* des_keysched_t)(uint64_t *, const void *)
Definition intel-ipsec-mb.h:719
+
keyexp_t
void(* keyexp_t)(const void *, void *, void *)
Definition intel-ipsec-mb.h:714
+
snow3g_f8_8_buffer_t
void(* snow3g_f8_8_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:865
+
aes_gcm_dec_192_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-192 encryption message.
+
imb_get_strerror
IMB_DLL_EXPORT const char * imb_get_strerror(int errnum)
API to get description for errnum.
+
aes_cmac_subkey_gen_avx
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1, void *key2)
+
aes_gcm_init_var_iv_t
void(* aes_gcm_init_var_iv_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t, const uint8_t *, const uint64_t)
Definition intel-ipsec-mb.h:736
+
aes_keyexp_128_enc_avx512
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key, void *enc_exp_keys)
+
aes_cmac_subkey_gen_avx2
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1, void *key2)
+
get_completed_job_avx512
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx512(IMB_MGR *state)
Get next completed job.
+
snow3g_f8_n_buffer_t
void(* snow3g_f8_n_buffer_t)(const snow3g_key_schedule_t *, const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
Definition intel-ipsec-mb.h:883
+
IMB_SGL_STATE
IMB_SGL_STATE
Definition intel-ipsec-mb.h:355
+
IMB_SGL_UPDATE
@ IMB_SGL_UPDATE
Definition intel-ipsec-mb.h:357
+
IMB_SGL_ALL
@ IMB_SGL_ALL
Definition intel-ipsec-mb.h:359
+
IMB_SGL_COMPLETE
@ IMB_SGL_COMPLETE
Definition intel-ipsec-mb.h:358
+
IMB_SGL_INIT
@ IMB_SGL_INIT
Definition intel-ipsec-mb.h:356
+
aes_gcm_pre_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-192 key data.
+
aes_gcm_precomp_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data)
Precomputation of AES-GCM-192 HashKey constants.
+
des_cfb_one
IMB_DLL_EXPORT void des_cfb_one(void *out, const void *in, const uint64_t *iv, const uint64_t *ks, const int len)
DES-CFB Encrypt/Decrypt up to one block.
+
init_mb_mgr_avx2
IMB_DLL_EXPORT void init_mb_mgr_avx2(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
aes_gcm_enc_dec_t
void(* aes_gcm_enc_dec_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, uint64_t, const uint8_t *, uint8_t const *, uint64_t, uint8_t *, uint64_t)
Definition intel-ipsec-mb.h:722
+
imb_get_feature_flags
IMB_DLL_EXPORT uint64_t imb_get_feature_flags(void)
Retrieves the bitmask with the features supported by the library, without having to allocate/initiali...
+
aes_keyexp_192_enc_avx512
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key, void *enc_exp_keys)
+
submit_job_nocheck_avx2
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx2(IMB_MGR *state)
Submit job for processing without validating.
+
aes_gcm_precomp_192_sse
IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data)
Precomputation of AES-GCM-192 HashKey constants.
+
aes_gcm_dec_256_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-256 encryption message.
+
aes_cmac_subkey_gen_avx512
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1, void *key2)
+
IMB_GCM_KEY_SETS
#define IMB_GCM_KEY_SETS
Definition intel-ipsec-mb.h:629
+
aes_keyexp_256_enc_avx2
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key, void *enc_exp_keys)
+
aes_gcm_init_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
+
submit_job_avx512
IMB_DLL_EXPORT IMB_JOB * submit_job_avx512(IMB_MGR *state)
Submit job for processing after validating.
+
aes_keyexp_192_sse
IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
submit_job_sse
IMB_DLL_EXPORT IMB_JOB * submit_job_sse(IMB_MGR *state)
Submit job for processing after validating.
+
aes_gcm_dec_256_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-256 encryption message.
+
aes_gcm_dec_128_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-128 encryption message.
+
imb_get_errno
IMB_DLL_EXPORT int imb_get_errno(IMB_MGR *mb_mgr)
API to get error status.
+
aes_gcm_init_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
+
md5_one_block_avx
IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest)
+
aes_gcm_enc_128_update_sse
IMB_DLL_EXPORT void aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-128 encryption message.
+
snow3g_f8_1_buffer_bit_t
void(* snow3g_f8_1_buffer_bit_t)(const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t, const uint32_t)
Definition intel-ipsec-mb.h:848
+
kasumi_init_f8_key_sched_t
int(* kasumi_init_f8_key_sched_t)(const void *, kasumi_key_sched_t *)
Definition intel-ipsec-mb.h:829
+
zuc_eea3_4_buffer_t
void(* zuc_eea3_4_buffer_t)(const void *const *, const void *const *, const void *const *, void **, const uint32_t *)
Definition intel-ipsec-mb.h:773
+
aes_gcm_enc_256_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-256 encryption message.
+
queue_size_avx
IMB_DLL_EXPORT uint32_t queue_size_avx(IMB_MGR *state)
Get number of jobs queued to be processed.
+
queue_size_sse
IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state)
Get number of jobs queued to be processed.
+
kasumi_f8_1_buffer_bit_t
void(* kasumi_f8_1_buffer_bit_t)(const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t, const uint32_t)
Definition intel-ipsec-mb.h:793
+
kasumi_key_sched_t
struct kasumi_key_sched_s kasumi_key_sched_t
+
IMB_HASH_ALG
IMB_HASH_ALG
Definition intel-ipsec-mb.h:293
+
IMB_AUTH_AES_CMAC_256
@ IMB_AUTH_AES_CMAC_256
Definition intel-ipsec-mb.h:320
+
IMB_AUTH_AES_XCBC
@ IMB_AUTH_AES_XCBC
Definition intel-ipsec-mb.h:299
+
IMB_AUTH_ZUC_EIA3_BITLEN
@ IMB_AUTH_ZUC_EIA3_BITLEN
Definition intel-ipsec-mb.h:313
+
IMB_AUTH_AES_GMAC_128
@ IMB_AUTH_AES_GMAC_128
Definition intel-ipsec-mb.h:317
+
IMB_AUTH_CRC16_X25
@ IMB_AUTH_CRC16_X25
Definition intel-ipsec-mb.h:332
+
IMB_AUTH_NULL
@ IMB_AUTH_NULL
Definition intel-ipsec-mb.h:301
+
IMB_AUTH_PON_CRC_BIP
@ IMB_AUTH_PON_CRC_BIP
Definition intel-ipsec-mb.h:312
+
IMB_AUTH_SHA_224
@ IMB_AUTH_SHA_224
Definition intel-ipsec-mb.h:307
+
IMB_AUTH_CRC24_LTE_A
@ IMB_AUTH_CRC24_LTE_A
Definition intel-ipsec-mb.h:330
+
IMB_AUTH_CRC32_ETHERNET_FCS
@ IMB_AUTH_CRC32_ETHERNET_FCS
Definition intel-ipsec-mb.h:327
+
IMB_AUTH_KASUMI_UIA1
@ IMB_AUTH_KASUMI_UIA1
Definition intel-ipsec-mb.h:316
+
IMB_AUTH_HMAC_SHA_1
@ IMB_AUTH_HMAC_SHA_1
Definition intel-ipsec-mb.h:294
+
IMB_AUTH_SHA_256
@ IMB_AUTH_SHA_256
Definition intel-ipsec-mb.h:308
+
IMB_AUTH_SNOW3G_UIA2_BITLEN
@ IMB_AUTH_SNOW3G_UIA2_BITLEN
Definition intel-ipsec-mb.h:315
+
IMB_AUTH_SNOW_V_AEAD
@ IMB_AUTH_SNOW_V_AEAD
Definition intel-ipsec-mb.h:325
+
IMB_AUTH_CRC32_SCTP
@ IMB_AUTH_CRC32_SCTP
Definition intel-ipsec-mb.h:328
+
IMB_AUTH_NUM
@ IMB_AUTH_NUM
Definition intel-ipsec-mb.h:340
+
IMB_AUTH_HMAC_SHA_256
@ IMB_AUTH_HMAC_SHA_256
Definition intel-ipsec-mb.h:296
+
IMB_AUTH_GHASH
@ IMB_AUTH_GHASH
Definition intel-ipsec-mb.h:339
+
IMB_AUTH_CUSTOM
@ IMB_AUTH_CUSTOM
Definition intel-ipsec-mb.h:303
+
IMB_AUTH_CHACHA20_POLY1305_SGL
@ IMB_AUTH_CHACHA20_POLY1305_SGL
Definition intel-ipsec-mb.h:323
+
IMB_AUTH_CRC24_LTE_B
@ IMB_AUTH_CRC24_LTE_B
Definition intel-ipsec-mb.h:331
+
IMB_AUTH_ZUC256_EIA3_BITLEN
@ IMB_AUTH_ZUC256_EIA3_BITLEN
Definition intel-ipsec-mb.h:324
+
IMB_AUTH_AES_CCM
@ IMB_AUTH_AES_CCM
Definition intel-ipsec-mb.h:304
+
IMB_AUTH_CRC6_IUUP_HEADER
@ IMB_AUTH_CRC6_IUUP_HEADER
Definition intel-ipsec-mb.h:338
+
IMB_AUTH_POLY1305
@ IMB_AUTH_POLY1305
Definition intel-ipsec-mb.h:321
+
IMB_AUTH_MD5
@ IMB_AUTH_MD5
Definition intel-ipsec-mb.h:300
+
IMB_AUTH_SHA_512
@ IMB_AUTH_SHA_512
Definition intel-ipsec-mb.h:310
+
IMB_AUTH_AES_GMAC
@ IMB_AUTH_AES_GMAC
Definition intel-ipsec-mb.h:302
+
IMB_AUTH_CRC16_FP_DATA
@ IMB_AUTH_CRC16_FP_DATA
Definition intel-ipsec-mb.h:333
+
IMB_AUTH_AES_CMAC_BITLEN
@ IMB_AUTH_AES_CMAC_BITLEN
Definition intel-ipsec-mb.h:311
+
IMB_AUTH_CRC11_FP_HEADER
@ IMB_AUTH_CRC11_FP_HEADER
Definition intel-ipsec-mb.h:334
+
IMB_AUTH_CHACHA20_POLY1305
@ IMB_AUTH_CHACHA20_POLY1305
Definition intel-ipsec-mb.h:322
+
IMB_AUTH_CRC7_FP_HEADER
@ IMB_AUTH_CRC7_FP_HEADER
Definition intel-ipsec-mb.h:337
+
IMB_AUTH_CRC10_IUUP_DATA
@ IMB_AUTH_CRC10_IUUP_DATA
Definition intel-ipsec-mb.h:335
+
IMB_AUTH_HMAC_SHA_224
@ IMB_AUTH_HMAC_SHA_224
Definition intel-ipsec-mb.h:295
+
IMB_AUTH_CRC32_WIMAX_OFDMA_DATA
@ IMB_AUTH_CRC32_WIMAX_OFDMA_DATA
Definition intel-ipsec-mb.h:329
+
IMB_AUTH_HMAC_SHA_512
@ IMB_AUTH_HMAC_SHA_512
Definition intel-ipsec-mb.h:298
+
IMB_AUTH_GCM_SGL
@ IMB_AUTH_GCM_SGL
Definition intel-ipsec-mb.h:326
+
IMB_AUTH_SHA_1
@ IMB_AUTH_SHA_1
Definition intel-ipsec-mb.h:306
+
IMB_AUTH_AES_GMAC_192
@ IMB_AUTH_AES_GMAC_192
Definition intel-ipsec-mb.h:318
+
IMB_AUTH_DOCSIS_CRC32
@ IMB_AUTH_DOCSIS_CRC32
Definition intel-ipsec-mb.h:314
+
IMB_AUTH_CRC8_WIMAX_OFDMA_HCS
@ IMB_AUTH_CRC8_WIMAX_OFDMA_HCS
Definition intel-ipsec-mb.h:336
+
IMB_AUTH_AES_GMAC_256
@ IMB_AUTH_AES_GMAC_256
Definition intel-ipsec-mb.h:319
+
IMB_AUTH_HMAC_SHA_384
@ IMB_AUTH_HMAC_SHA_384
Definition intel-ipsec-mb.h:297
+
IMB_AUTH_AES_CMAC
@ IMB_AUTH_AES_CMAC
Definition intel-ipsec-mb.h:305
+
IMB_AUTH_SHA_384
@ IMB_AUTH_SHA_384
Definition intel-ipsec-mb.h:309
+
aes_keyexp_128_enc_avx2
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key, void *enc_exp_keys)
+
aes_gcm_enc_256_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-256 encryption message.
+
submit_job_avx2
IMB_DLL_EXPORT IMB_JOB * submit_job_avx2(IMB_MGR *state)
Submit job for processing after validating.
+
imb_quic_hp_aes_ecb
IMB_DLL_EXPORT void imb_quic_hp_aes_ecb(IMB_MGR *state, const void *exp_key_data, void *dst_ptr_array[], const void *const src_ptr_array[], const uint64_t num_packets, const IMB_KEY_SIZE_BYTES key_size)
Batch of AES-ECB encrypt/decrypt operations with the same key.
+
kasumi_f8_1_buffer_t
void(* kasumi_f8_1_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:790
+
queue_size_avx2
IMB_DLL_EXPORT uint32_t queue_size_avx2(IMB_MGR *state)
Get number of jobs queued to be processed.
+
kasumi_f9_1_buffer_t
void(* kasumi_f9_1_buffer_t)(const kasumi_key_sched_t *, const void *, const uint32_t, void *)
Definition intel-ipsec-mb.h:826
+
kasumi_init_f9_key_sched_t
int(* kasumi_init_f9_key_sched_t)(const void *, kasumi_key_sched_t *)
Definition intel-ipsec-mb.h:831
+
md5_one_block_avx2
IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest)
+
aes_gcm_enc_dec_finalize_t
void(* aes_gcm_enc_dec_finalize_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint64_t)
Definition intel-ipsec-mb.h:743
+
aes_gcm_dec_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Decryption.
+
aes_gcm_init_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
+
aes_gcm_dec_128_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-128 encryption message.
+
imb_quic_aes_gcm
IMB_DLL_EXPORT void imb_quic_aes_gcm(IMB_MGR *state, const struct gcm_key_data *key_data, const IMB_KEY_SIZE_BYTES key_size, const IMB_CIPHER_DIRECTION cipher_dir, void *dst_ptr_array[], const void *const src_ptr_array[], const uint64_t len_array[], const void *const iv_ptr_array[], const void *const aad_ptr_array[], const uint64_t aad_len, void *tag_ptr_array[], const uint64_t tag_len, const uint64_t num_packets)
Batch of GCM encrypt/decrypt operations with the same key.
+
snow3g_f9_iv_gen
IMB_DLL_EXPORT int snow3g_f9_iv_gen(const uint32_t count, const uint32_t fresh, const uint8_t dir, void *iv_ptr)
Generation of SNOW3G F9 Initialization Vector.
+
submit_job_avx
IMB_DLL_EXPORT IMB_JOB * submit_job_avx(IMB_MGR *state)
Submit job for processing after validating.
+
zuc_eia3_1_buffer_t
void(* zuc_eia3_1_buffer_t)(const void *, const void *, const void *, const uint32_t, uint32_t *)
Definition intel-ipsec-mb.h:781
+
flush_job_sse
IMB_DLL_EXPORT IMB_JOB * flush_job_sse(IMB_MGR *state)
Force processing until next job in queue is completed.
+
IMB_DLL_EXPORT
#define IMB_DLL_EXPORT
Definition intel-ipsec-mb.h:86
+
queue_size_t
uint32_t(* queue_size_t)(struct IMB_MGR *)
Definition intel-ipsec-mb.h:700
+
init_mb_mgr_avx512
IMB_DLL_EXPORT void init_mb_mgr_avx512(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
des_key_schedule
IMB_DLL_EXPORT int des_key_schedule(uint64_t *ks, const void *key)
DES key schedule set up.
+
aes_gcm_init_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
+
aes_gcm_enc_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Encryption.
+
aes_gcm_dec_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Decryption.
+
aes_keyexp_128_avx512
IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
get_completed_job_avx
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx(IMB_MGR *state)
Get next completed job.
+
md5_one_block_sse
IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest)
+
aes_keyexp_256_enc_avx512
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key, void *enc_exp_keys)
+
IMB_ERR
IMB_ERR
Definition intel-ipsec-mb.h:192
+
IMB_ERR_NULL_JOB
@ IMB_ERR_NULL_JOB
Definition intel-ipsec-mb.h:240
+
IMB_ERR_JOB_NULL_SGL_CTX
@ IMB_ERR_JOB_NULL_SGL_CTX
Definition intel-ipsec-mb.h:212
+
IMB_ERR_AAD_LEN
@ IMB_ERR_AAD_LEN
Definition intel-ipsec-mb.h:227
+
IMB_ERR_NULL_CTX
@ IMB_ERR_NULL_CTX
Definition intel-ipsec-mb.h:230
+
IMB_ERR_JOB_NULL_IV
@ IMB_ERR_JOB_NULL_IV
Definition intel-ipsec-mb.h:198
+
IMB_ERR_JOB_NULL_HMAC_IPAD
@ IMB_ERR_JOB_NULL_HMAC_IPAD
Definition intel-ipsec-mb.h:233
+
IMB_ERR_SELFTEST
@ IMB_ERR_SELFTEST
Definition intel-ipsec-mb.h:245
+
IMB_ERR_NULL_DST
@ IMB_ERR_NULL_DST
Definition intel-ipsec-mb.h:216
+
IMB_ERR_JOB_AUTH_TAG_LEN
@ IMB_ERR_JOB_AUTH_TAG_LEN
Definition intel-ipsec-mb.h:205
+
IMB_ERR_NULL_AUTH
@ IMB_ERR_NULL_AUTH
Definition intel-ipsec-mb.h:220
+
IMB_ERR_JOB_SRC_OFFSET
@ IMB_ERR_JOB_SRC_OFFSET
Definition intel-ipsec-mb.h:207
+
IMB_ERR_SRC_OFFSET
@ IMB_ERR_SRC_OFFSET
Definition intel-ipsec-mb.h:228
+
IMB_ERR_JOB_NULL_KEY
@ IMB_ERR_JOB_NULL_KEY
Definition intel-ipsec-mb.h:197
+
IMB_ERR_NULL_SRC
@ IMB_ERR_NULL_SRC
Definition intel-ipsec-mb.h:215
+
IMB_ERR_JOB_AAD_LEN
@ IMB_ERR_JOB_AAD_LEN
Definition intel-ipsec-mb.h:206
+
IMB_ERR_JOB_IV_LEN
@ IMB_ERR_JOB_IV_LEN
Definition intel-ipsec-mb.h:203
+
IMB_ERR_NULL_EXP_KEY
@ IMB_ERR_NULL_EXP_KEY
Definition intel-ipsec-mb.h:218
+
IMB_ERR_JOB_NULL_AAD
@ IMB_ERR_JOB_NULL_AAD
Definition intel-ipsec-mb.h:200
+
IMB_ERR_MIN
@ IMB_ERR_MIN
Definition intel-ipsec-mb.h:193
+
IMB_ERR_BURST_SUITE_ID
@ IMB_ERR_BURST_SUITE_ID
Definition intel-ipsec-mb.h:246
+
IMB_ERR_NO_AESNI_EMU
@ IMB_ERR_NO_AESNI_EMU
Definition intel-ipsec-mb.h:231
+
IMB_ERR_JOB_NULL_SRC
@ IMB_ERR_JOB_NULL_SRC
Definition intel-ipsec-mb.h:195
+
IMB_ERR_QUEUE_SPACE
@ IMB_ERR_QUEUE_SPACE
Definition intel-ipsec-mb.h:241
+
IMB_ERR_JOB_NULL_AUTH
@ IMB_ERR_JOB_NULL_AUTH
Definition intel-ipsec-mb.h:199
+
IMB_ERR_JOB_NULL_DST
@ IMB_ERR_JOB_NULL_DST
Definition intel-ipsec-mb.h:196
+
IMB_ERR_CIPH_MODE
@ IMB_ERR_CIPH_MODE
Definition intel-ipsec-mb.h:209
+
IMB_ERR_NULL_KEY
@ IMB_ERR_NULL_KEY
Definition intel-ipsec-mb.h:217
+
IMB_ERR_JOB_NULL_XCBC_K1_EXP
@ IMB_ERR_JOB_NULL_XCBC_K1_EXP
Definition intel-ipsec-mb.h:234
+
IMB_ERR_JOB_AUTH_LEN
@ IMB_ERR_JOB_AUTH_LEN
Definition intel-ipsec-mb.h:202
+
IMB_ERR_AUTH_TAG_LEN
@ IMB_ERR_AUTH_TAG_LEN
Definition intel-ipsec-mb.h:226
+
IMB_ERR_JOB_NULL_GHASH_INIT_TAG
@ IMB_ERR_JOB_NULL_GHASH_INIT_TAG
Definition intel-ipsec-mb.h:238
+
IMB_ERR_HASH_ALGO
@ IMB_ERR_HASH_ALGO
Definition intel-ipsec-mb.h:210
+
IMB_ERR_KEY_LEN
@ IMB_ERR_KEY_LEN
Definition intel-ipsec-mb.h:225
+
IMB_ERR_JOB_NULL_AUTH_KEY
@ IMB_ERR_JOB_NULL_AUTH_KEY
Definition intel-ipsec-mb.h:211
+
IMB_ERR_JOB_NULL_NEXT_IV
@ IMB_ERR_JOB_NULL_NEXT_IV
Definition intel-ipsec-mb.h:213
+
IMB_ERR_NULL_BURST
@ IMB_ERR_NULL_BURST
Definition intel-ipsec-mb.h:242
+
IMB_ERR_NULL_MBMGR
@ IMB_ERR_NULL_MBMGR
Definition intel-ipsec-mb.h:194
+
IMB_ERR_NULL_AUTH_KEY
@ IMB_ERR_NULL_AUTH_KEY
Definition intel-ipsec-mb.h:229
+
IMB_ERR_IV_LEN
@ IMB_ERR_IV_LEN
Definition intel-ipsec-mb.h:224
+
IMB_ERR_CIPH_LEN
@ IMB_ERR_CIPH_LEN
Definition intel-ipsec-mb.h:222
+
IMB_ERR_MISSING_CPUFLAGS_INIT_MGR
@ IMB_ERR_MISSING_CPUFLAGS_INIT_MGR
Definition intel-ipsec-mb.h:239
+
IMB_ERR_MAX
@ IMB_ERR_MAX
Definition intel-ipsec-mb.h:248
+
IMB_ERR_JOB_CHAIN_ORDER
@ IMB_ERR_JOB_CHAIN_ORDER
Definition intel-ipsec-mb.h:208
+
IMB_ERR_JOB_NULL_HMAC_OPAD
@ IMB_ERR_JOB_NULL_HMAC_OPAD
Definition intel-ipsec-mb.h:232
+
IMB_ERR_JOB_CIPH_DIR
@ IMB_ERR_JOB_CIPH_DIR
Definition intel-ipsec-mb.h:237
+
IMB_ERR_JOB_PON_PLI
@ IMB_ERR_JOB_PON_PLI
Definition intel-ipsec-mb.h:214
+
IMB_ERR_JOB_NULL_XCBC_K2
@ IMB_ERR_JOB_NULL_XCBC_K2
Definition intel-ipsec-mb.h:235
+
IMB_ERR_NULL_AAD
@ IMB_ERR_NULL_AAD
Definition intel-ipsec-mb.h:221
+
IMB_ERR_JOB_KEY_LEN
@ IMB_ERR_JOB_KEY_LEN
Definition intel-ipsec-mb.h:204
+
IMB_ERR_JOB_CIPH_LEN
@ IMB_ERR_JOB_CIPH_LEN
Definition intel-ipsec-mb.h:201
+
IMB_ERR_BURST_SIZE
@ IMB_ERR_BURST_SIZE
Definition intel-ipsec-mb.h:243
+
IMB_ERR_BURST_OOO
@ IMB_ERR_BURST_OOO
Definition intel-ipsec-mb.h:244
+
IMB_ERR_JOB_NULL_XCBC_K3
@ IMB_ERR_JOB_NULL_XCBC_K3
Definition intel-ipsec-mb.h:236
+
IMB_ERR_NULL_IV
@ IMB_ERR_NULL_IV
Definition intel-ipsec-mb.h:219
+
IMB_ERR_AUTH_LEN
@ IMB_ERR_AUTH_LEN
Definition intel-ipsec-mb.h:223
+
aes_gcm_enc_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Encryption.
+
hec_32_t
uint32_t(* hec_32_t)(const uint8_t *)
Definition intel-ipsec-mb.h:905
+
aes_gcm_dec_192_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-192 encryption message.
+
hec_64_t
uint64_t(* hec_64_t)(const uint8_t *)
Definition intel-ipsec-mb.h:906
+
aes_gcm_pre_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-256 key data.
+
cmac_subkey_gen_t
void(* cmac_subkey_gen_t)(const void *, void *, void *)
Definition intel-ipsec-mb.h:715
+
aes_gcm_dec_192_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-192 encryption message.
+
aes_gcm_enc_128_sse
IMB_DLL_EXPORT void aes_gcm_enc_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Encryption.
+
hash_one_block_t
void(* hash_one_block_t)(const void *, void *)
Definition intel-ipsec-mb.h:716
+
burst_fn_t
uint32_t(* burst_fn_t)(struct IMB_MGR *, const uint32_t, struct IMB_JOB **)
Definition intel-ipsec-mb.h:701
+
aes_gcm_precomp_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data)
Precomputation of AES-GCM-192 HashKey constants.
+
IMB_MAX_JOBS
#define IMB_MAX_JOBS
Definition intel-ipsec-mb.h:978
+
aes_gcm_dec_256_update_sse
IMB_DLL_EXPORT void aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-256 encryption message.
+
aes_gcm_precomp_t
void(* aes_gcm_precomp_t)(struct gcm_key_data *)
Definition intel-ipsec-mb.h:746
+
imb_hmac_ipad_opad
IMB_DLL_EXPORT void imb_hmac_ipad_opad(struct IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, const void *pkey, const size_t key_len, void *ipad_hash, void *opad_hash)
Ipad Opad padding for HMAC.
+
aes_gcm_pre_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-192 key data.
+
aes_gcm_enc_256_update_sse
IMB_DLL_EXPORT void aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-256 encryption message.
+
alloc_mb_mgr
IMB_DLL_EXPORT IMB_MGR * alloc_mb_mgr(uint64_t flags)
Allocates memory for multi-buffer manager instance.
+
aes_gmac_update_t
void(* aes_gmac_update_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
Definition intel-ipsec-mb.h:752
+
aes_keyexp_192_avx2
IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
xcbc_keyexp_t
void(* xcbc_keyexp_t)(const void *, void *, void *, void *)
Definition intel-ipsec-mb.h:718
+
aes_gcm_enc_128_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-128 encryption message.
+
flush_job_avx
IMB_DLL_EXPORT IMB_JOB * flush_job_avx(IMB_MGR *state)
Force processing until next job in queue is completed.
+
IMB_CIPHER_DIRECTION
IMB_CIPHER_DIRECTION
Definition intel-ipsec-mb.h:288
+
IMB_DIR_DECRYPT
@ IMB_DIR_DECRYPT
Definition intel-ipsec-mb.h:290
+
IMB_DIR_ENCRYPT
@ IMB_DIR_ENCRYPT
Definition intel-ipsec-mb.h:289
+
aes_gcm_precomp_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data)
Precomputation of AES-GCM-256 HashKey constants.
+
aes_gcm_pre_192_sse
IMB_DLL_EXPORT void aes_gcm_pre_192_sse(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-192 key data.
+
aes_gcm_dec_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Decryption.
+
kasumi_f9_1_buffer_user_t
void(* kasumi_f9_1_buffer_user_t)(const kasumi_key_sched_t *, const uint64_t, const void *, const uint32_t, void *, const uint32_t)
Definition intel-ipsec-mb.h:822
+
aes_gcm_dec_128_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-128 encryption message.
+
aes_gcm_init_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
+
aes_xcbc_expand_key_avx2
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp, void *k2, void *k3)
+
submit_cipher_burst_t
uint32_t(* submit_cipher_burst_t)(struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size)
Definition intel-ipsec-mb.h:704
+
aes_gcm_enc_dec_iv_t
void(* aes_gcm_enc_dec_iv_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, const uint64_t, const uint8_t *, uint8_t const *, const uint64_t, uint8_t *, const uint64_t, const uint64_t)
Definition intel-ipsec-mb.h:727
+
flush_job_avx512
IMB_DLL_EXPORT IMB_JOB * flush_job_avx512(IMB_MGR *state)
Force processing until next job in queue is completed.
+
submit_job_nocheck_avx
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx(IMB_MGR *state)
Submit job for processing without validating.
+
aes_keyexp_128_enc_avx
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key, void *enc_exp_keys)
+
md5_one_block_avx512
IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest)
+
imb_get_mb_mgr_size
IMB_DLL_EXPORT size_t imb_get_mb_mgr_size(void)
Calculates necessary memory size for IMB_MGR.
+
kasumi_f9_iv_gen
IMB_DLL_EXPORT int kasumi_f9_iv_gen(const uint32_t count, const uint32_t fresh, void *iv_ptr)
Generation of KASUMI F9 Initialization Vector.
+
get_completed_job_t
IMB_JOB *(* get_completed_job_t)(struct IMB_MGR *)
Definition intel-ipsec-mb.h:698
+
aes_gcm_init_128_sse
IMB_DLL_EXPORT void aes_gcm_init_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
+
aes_gcm_dec_192_sse
IMB_DLL_EXPORT void aes_gcm_dec_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Decryption.
+
kasumi_f8_4_buffer_t
void(* kasumi_f8_4_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:810
+
IMB_ARCH
IMB_ARCH
Definition intel-ipsec-mb.h:119
+
IMB_ARCH_AVX512
@ IMB_ARCH_AVX512
Definition intel-ipsec-mb.h:125
+
IMB_ARCH_NONE
@ IMB_ARCH_NONE
Definition intel-ipsec-mb.h:120
+
IMB_ARCH_SSE
@ IMB_ARCH_SSE
Definition intel-ipsec-mb.h:122
+
IMB_ARCH_NOAESNI
@ IMB_ARCH_NOAESNI
Definition intel-ipsec-mb.h:121
+
IMB_ARCH_AVX2
@ IMB_ARCH_AVX2
Definition intel-ipsec-mb.h:124
+
IMB_ARCH_AVX
@ IMB_ARCH_AVX
Definition intel-ipsec-mb.h:123
+
IMB_ARCH_NUM
@ IMB_ARCH_NUM
Definition intel-ipsec-mb.h:126
+
aes_gcm_dec_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Decryption.
+
kasumi_key_sched_size_t
size_t(* kasumi_key_sched_size_t)(void)
Definition intel-ipsec-mb.h:833
+
snow3g_key_sched_size_t
size_t(* snow3g_key_sched_size_t)(void)
Definition intel-ipsec-mb.h:903
+
get_completed_job_avx2
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx2(IMB_MGR *state)
Get next completed job.
+
aes_ecb_quic_t
void(* aes_ecb_quic_t)(const void *, const void *, void *out, uint64_t)
Definition intel-ipsec-mb.h:910
+
aes_gcm_enc_256_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-256 encryption message.
+
imb_set_session
IMB_DLL_EXPORT uint32_t imb_set_session(IMB_MGR *state, IMB_JOB *job)
Sets up suite_id and session_id fields for selected cipher suite in provided job structure.
+
zuc_eia3_n_buffer_t
void(* zuc_eia3_n_buffer_t)(const void *const *, const void *const *, const void *const *, const uint32_t *, uint32_t **, const uint32_t)
Definition intel-ipsec-mb.h:784
+
aes_gcm_dec_128_sse
IMB_DLL_EXPORT void aes_gcm_dec_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Decryption.
+
aes_gcm_precomp_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data)
Precomputation of AES-GCM-128 HashKey constants.
+
aes_gcm_enc_192_sse
IMB_DLL_EXPORT void aes_gcm_enc_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Encryption.
+
aes_gcm_init_t
void(* aes_gcm_init_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, uint8_t const *, uint64_t)
Definition intel-ipsec-mb.h:733
+
aes_gcm_dec_256_finalize_sse
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-256 encryption message.
+
snow3g_init_key_sched_t
int(* snow3g_init_key_sched_t)(const void *, snow3g_key_schedule_t *)
Definition intel-ipsec-mb.h:900
+
snow3g_f8_4_buffer_t
void(* snow3g_f8_4_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:857
+
aes_gcm_dec_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Decryption.
+
flush_job_avx2
IMB_DLL_EXPORT IMB_JOB * flush_job_avx2(IMB_MGR *state)
Force processing until next job in queue is completed.
+
aes_gcm_enc_128_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-128 encryption message.
+
zuc_eia3_iv_gen
IMB_DLL_EXPORT int zuc_eia3_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of ZUC-EIA3 Initialization Vector.
+
snow3g_key_schedule_t
struct snow3g_key_schedule_s snow3g_key_schedule_t
+
flush_job_t
IMB_JOB *(* flush_job_t)(struct IMB_MGR *)
Definition intel-ipsec-mb.h:699
+
snow3g_f9_1_buffer_t
void(* snow3g_f9_1_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const uint64_t, void *)
Definition intel-ipsec-mb.h:896
+
aes_cfb_t
void(* aes_cfb_t)(void *, const void *, const void *, const void *, uint64_t)
Definition intel-ipsec-mb.h:720
+
aes_keyexp_128_avx
IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
imb_clear_mem
IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size)
Force clearing/zeroing of memory.
+
aes_gcm_precomp_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data)
Precomputation of AES-GCM-128 HashKey constants.
+
imb_get_version
IMB_DLL_EXPORT unsigned imb_get_version(void)
Get library version in numerical format.
+
init_mb_mgr_avx
IMB_DLL_EXPORT void init_mb_mgr_avx(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
aes_xcbc_expand_key_sse
IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp, void *k2, void *k3)
+
init_mb_mgr_t
void(* init_mb_mgr_t)(struct IMB_MGR *)
Definition intel-ipsec-mb.h:695
+
kasumi_f8_2_buffer_t
void(* kasumi_f8_2_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const uint64_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition intel-ipsec-mb.h:797
+
aes_keyexp_256_avx2
IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_gcm_enc_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Encryption.
+
aes_gmac_finalize_t
void(* aes_gmac_finalize_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint64_t)
Definition intel-ipsec-mb.h:755
+
aes_gcm_enc_128_finalize_sse
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-128 encryption message.
+
aes_gcm_enc_dec_update_t
void(* aes_gcm_enc_dec_update_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint8_t *, uint64_t)
Definition intel-ipsec-mb.h:740
+
aes_gcm_dec_192_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-192 encryption message.
+
aes_keyexp_256_enc_sse
IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key, void *enc_exp_keys)
+
hash_fn_t
void(* hash_fn_t)(const void *, const uint64_t, void *)
Definition intel-ipsec-mb.h:717
+
aes_gcm_enc_256_finalize_sse
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-256 encryption message.
+
IMB_STATUS
IMB_STATUS
Definition intel-ipsec-mb.h:178
+
IMB_STATUS_ERROR
@ IMB_STATUS_ERROR
Definition intel-ipsec-mb.h:186
+
IMB_STATUS_COMPLETED
@ IMB_STATUS_COMPLETED
Definition intel-ipsec-mb.h:182
+
IMB_STATUS_INTERNAL_ERROR
@ IMB_STATUS_INTERNAL_ERROR
Definition intel-ipsec-mb.h:185
+
IMB_STATUS_COMPLETED_CIPHER
@ IMB_STATUS_COMPLETED_CIPHER
Definition intel-ipsec-mb.h:180
+
IMB_STATUS_COMPLETED_AUTH
@ IMB_STATUS_COMPLETED_AUTH
Definition intel-ipsec-mb.h:181
+
IMB_STATUS_BEING_PROCESSED
@ IMB_STATUS_BEING_PROCESSED
Definition intel-ipsec-mb.h:179
+
IMB_STATUS_INVALID_ARGS
@ IMB_STATUS_INVALID_ARGS
Definition intel-ipsec-mb.h:184
+
aes_gcm_dec_256_sse
IMB_DLL_EXPORT void aes_gcm_dec_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Decryption.
+
aes_xcbc_expand_key_avx
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp, void *k2, void *k3)
+
IMB_JOB
Definition intel-ipsec-mb.h:390
+
IMB_JOB::cipher_fields
union IMB_JOB::@11 cipher_fields
+
IMB_JOB::ctx
struct gcm_context_data * ctx
Definition intel-ipsec-mb.h:468
+
IMB_JOB::GMAC
struct IMB_JOB::@10::_AES_GMAC_specific_fields GMAC
+
IMB_JOB::KASUMI_UIA1
struct IMB_JOB::@10::_KASUMI_UIA1_specific_fields KASUMI_UIA1
+
IMB_JOB::cipher_direction
IMB_CIPHER_DIRECTION cipher_direction
Definition intel-ipsec-mb.h:526
+
IMB_JOB::POLY1305
struct IMB_JOB::@10::_POLY1305_specific_fields POLY1305
+
IMB_JOB::num_sgl_io_segs
uint64_t num_sgl_io_segs
Definition intel-ipsec-mb.h:405
+
IMB_JOB::_init_tag
const void * _init_tag
Definition intel-ipsec-mb.h:500
+
IMB_JOB::sgl_io_segs
const struct IMB_SGL_IOV * sgl_io_segs
Definition intel-ipsec-mb.h:398
+
IMB_JOB::iv_len_in_bytes
uint64_t iv_len_in_bytes
Definition intel-ipsec-mb.h:430
+
IMB_JOB::SNOW3G_UIA2
struct IMB_JOB::@10::_SNOW3G_UIA2_specific_fields SNOW3G_UIA2
+
IMB_JOB::_iv23
const uint8_t * _iv23
Definition intel-ipsec-mb.h:476
+
IMB_JOB::_hashed_auth_key_xor_ipad
const uint8_t * _hashed_auth_key_xor_ipad
Definition intel-ipsec-mb.h:438
+
IMB_JOB::suite_id
void * suite_id[4]
Definition intel-ipsec-mb.h:550
+
IMB_JOB::XCBC
struct IMB_JOB::@10::_AES_XCBC_specific_fields XCBC
+
IMB_JOB::_key
const struct gcm_key_data * _key
Definition intel-ipsec-mb.h:490
+
IMB_JOB::sgl_state
IMB_SGL_STATE sgl_state
Definition intel-ipsec-mb.h:539
+
IMB_JOB::msg_len_to_hash_in_bytes
uint64_t msg_len_to_hash_in_bytes
Definition intel-ipsec-mb.h:424
+
IMB_JOB::chain_order
IMB_CHAIN_ORDER chain_order
Definition intel-ipsec-mb.h:528
+
IMB_JOB::msg_len_to_cipher_in_bits
uint64_t msg_len_to_cipher_in_bits
Definition intel-ipsec-mb.h:419
+
IMB_JOB::_iv
const void * _iv
Definition intel-ipsec-mb.h:482
+
IMB_JOB::CMAC
struct IMB_JOB::@10::_AES_CMAC_specific_fields CMAC
+
IMB_JOB::ctx
struct chacha20_poly1305_context_data * ctx
Definition intel-ipsec-mb.h:511
+
IMB_JOB::aad
const void * aad
Definition intel-ipsec-mb.h:454
+
IMB_JOB::user_data2
void * user_data2
Definition intel-ipsec-mb.h:532
+
IMB_JOB::next_iv
void * next_iv
Definition intel-ipsec-mb.h:545
+
IMB_JOB::u
union IMB_JOB::@10 u
+
IMB_JOB::cipher_start_offset_in_bits
uint64_t cipher_start_offset_in_bits
Definition intel-ipsec-mb.h:413
+
IMB_JOB::hash_func
int(* hash_func)(struct IMB_JOB *)
Definition intel-ipsec-mb.h:536
+
IMB_JOB::cipher_func
int(* cipher_func)(struct IMB_JOB *)
Definition intel-ipsec-mb.h:534
+
IMB_JOB::_hashed_auth_key_xor_opad
const uint8_t * _hashed_auth_key_xor_opad
Definition intel-ipsec-mb.h:441
+
IMB_JOB::auth_tag_output_len_in_bytes
uint64_t auth_tag_output_len_in_bytes
Definition intel-ipsec-mb.h:432
+
IMB_JOB::_k1_expanded
const uint32_t * _k1_expanded
Definition intel-ipsec-mb.h:446
+
IMB_JOB::cipher_mode
IMB_CIPHER_MODE cipher_mode
Definition intel-ipsec-mb.h:525
+
IMB_JOB::_key
const void * _key
Definition intel-ipsec-mb.h:480
+
IMB_JOB::_key
const uint8_t * _key
Definition intel-ipsec-mb.h:472
+
IMB_JOB::CBCS
struct IMB_JOB::@11::_CBCS_specific_fields CBCS
+
IMB_JOB::_skey2
const void * _skey2
Definition intel-ipsec-mb.h:462
+
IMB_JOB::_key_expanded
const void * _key_expanded
Definition intel-ipsec-mb.h:459
+
IMB_JOB::iv
const uint8_t * iv
Definition intel-ipsec-mb.h:429
+
IMB_JOB::msg_len_to_cipher_in_bytes
uint64_t msg_len_to_cipher_in_bytes
Definition intel-ipsec-mb.h:417
+
IMB_JOB::reserved
void * reserved
Definition intel-ipsec-mb.h:519
+
IMB_JOB::hash_alg
IMB_HASH_ALG hash_alg
Definition intel-ipsec-mb.h:527
+
IMB_JOB::aad_len_in_bytes
uint64_t aad_len_in_bytes
Definition intel-ipsec-mb.h:456
+
IMB_JOB::HMAC
struct IMB_JOB::@10::_HMAC_specific_fields HMAC
+
IMB_JOB::ZUC_EIA3
struct IMB_JOB::@10::_ZUC_EIA3_specific_fields ZUC_EIA3
+
IMB_JOB::session_id
uint32_t session_id
Definition intel-ipsec-mb.h:551
+
IMB_JOB::_skey1
const void * _skey1
Definition intel-ipsec-mb.h:461
+
IMB_JOB::user_data
void * user_data
Definition intel-ipsec-mb.h:531
+
IMB_JOB::key_len_in_bytes
uint64_t key_len_in_bytes
Definition intel-ipsec-mb.h:393
+
IMB_JOB::SNOW_V_AEAD
struct IMB_JOB::@10::_SNOW_V_AEAD_specific_fields SNOW_V_AEAD
+
IMB_JOB::dst
uint8_t * dst
Definition intel-ipsec-mb.h:402
+
IMB_JOB::auth_tag_output
uint8_t * auth_tag_output
Definition intel-ipsec-mb.h:431
+
IMB_JOB::enc_keys
const void * enc_keys
Definition intel-ipsec-mb.h:391
+
IMB_JOB::GCM
struct IMB_JOB::@10::_AES_GCM_specific_fields GCM
+
IMB_JOB::CHACHA20_POLY1305
struct IMB_JOB::@10::_CHACHA20_POLY1305_specific_fields CHACHA20_POLY1305
+
IMB_JOB::CCM
struct IMB_JOB::@10::_AES_CCM_specific_fields CCM
+
IMB_JOB::status
IMB_STATUS status
Definition intel-ipsec-mb.h:524
+
IMB_JOB::_k2
const uint8_t * _k2
Definition intel-ipsec-mb.h:448
+
IMB_JOB::cipher_start_src_offset_in_bytes
uint64_t cipher_start_src_offset_in_bytes
Definition intel-ipsec-mb.h:409
+
IMB_JOB::msg_len_to_hash_in_bits
uint64_t msg_len_to_hash_in_bits
Definition intel-ipsec-mb.h:426
+
IMB_JOB::_iv
const uint8_t * _iv
Definition intel-ipsec-mb.h:474
+
IMB_JOB::GHASH
struct IMB_JOB::@10::_GHASH_specific_fields GHASH
+
IMB_JOB::cipher_start_src_offset_in_bits
uint64_t cipher_start_src_offset_in_bits
Definition intel-ipsec-mb.h:411
+
IMB_JOB::dec_keys
const void * dec_keys
Definition intel-ipsec-mb.h:392
+
IMB_JOB::hash_start_src_offset_in_bytes
uint64_t hash_start_src_offset_in_bytes
Definition intel-ipsec-mb.h:422
+
IMB_JOB::src
const uint8_t * src
Definition intel-ipsec-mb.h:395
+
IMB_JOB::_k3
const uint8_t * _k3
Definition intel-ipsec-mb.h:450
+
IMB_MGR
Definition intel-ipsec-mb.h:980
+
IMB_MGR::gmac192_finalize
aes_gmac_finalize_t gmac192_finalize
Definition intel-ipsec-mb.h:1091
+
IMB_MGR::gcm192_init
aes_gcm_init_t gcm192_init
Definition intel-ipsec-mb.h:1028
+
IMB_MGR::gmac128_finalize
aes_gmac_finalize_t gmac128_finalize
Definition intel-ipsec-mb.h:1090
+
IMB_MGR::aes256_ccm_ooo
void * aes256_ccm_ooo
Definition intel-ipsec-mb.h:1164
+
IMB_MGR::crc24_lte_a
crc32_fn_t crc24_lte_a
Definition intel-ipsec-mb.h:1100
+
IMB_MGR::gmac256_update
aes_gmac_update_t gmac256_update
Definition intel-ipsec-mb.h:1089
+
IMB_MGR::hec_32
hec_32_t hec_32
Definition intel-ipsec-mb.h:1093
+
IMB_MGR::imb_errno
int imb_errno
Definition intel-ipsec-mb.h:988
+
IMB_MGR::aes_cmac_ooo
void * aes_cmac_ooo
Definition intel-ipsec-mb.h:1158
+
IMB_MGR::next_job
int next_job
Definition intel-ipsec-mb.h:1132
+
IMB_MGR::gmac128_init
aes_gmac_init_t gmac128_init
Definition intel-ipsec-mb.h:1084
+
IMB_MGR::reserved
uint64_t reserved[5]
Definition intel-ipsec-mb.h:985
+
IMB_MGR::gcm192_pre
aes_gcm_pre_t gcm192_pre
Definition intel-ipsec-mb.h:1046
+
IMB_MGR::snow3g_f8_4_buffer
snow3g_f8_4_buffer_t snow3g_f8_4_buffer
Definition intel-ipsec-mb.h:1069
+
IMB_MGR::gcm256_precomp
aes_gcm_precomp_t gcm256_precomp
Definition intel-ipsec-mb.h:1044
+
IMB_MGR::gcm128_pre
aes_gcm_pre_t gcm128_pre
Definition intel-ipsec-mb.h:1045
+
IMB_MGR::submit_hash_burst
submit_hash_burst_t submit_hash_burst
Definition intel-ipsec-mb.h:1121
+
IMB_MGR::flush_job
flush_job_t flush_job
Definition intel-ipsec-mb.h:1000
+
IMB_MGR::snow3g_f8_2_buffer
snow3g_f8_2_buffer_t snow3g_f8_2_buffer
Definition intel-ipsec-mb.h:1068
+
IMB_MGR::zuc_eea3_ooo
void * zuc_eea3_ooo
Definition intel-ipsec-mb.h:1159
+
IMB_MGR::gcm128_dec_update
aes_gcm_enc_dec_update_t gcm128_dec_update
Definition intel-ipsec-mb.h:1033
+
IMB_MGR::des3_dec_ooo
void * des3_dec_ooo
Definition intel-ipsec-mb.h:1146
+
IMB_MGR::keyexp_192
keyexp_t keyexp_192
Definition intel-ipsec-mb.h:1003
+
IMB_MGR::gcm128_dec
aes_gcm_enc_dec_t gcm128_dec
Definition intel-ipsec-mb.h:1024
+
IMB_MGR::submit_burst_nocheck
burst_fn_t submit_burst_nocheck
Definition intel-ipsec-mb.h:1117
+
IMB_MGR::get_next_job
get_next_job_t get_next_job
Definition intel-ipsec-mb.h:996
+
IMB_MGR::sha_256_ooo
void * sha_256_ooo
Definition intel-ipsec-mb.h:1170
+
IMB_MGR::snow3g_init_key_sched
snow3g_init_key_sched_t snow3g_init_key_sched
Definition intel-ipsec-mb.h:1075
+
IMB_MGR::gcm192_enc_update
aes_gcm_enc_dec_update_t gcm192_enc_update
Definition intel-ipsec-mb.h:1031
+
IMB_MGR::gmac256_finalize
aes_gmac_finalize_t gmac256_finalize
Definition intel-ipsec-mb.h:1092
+
IMB_MGR::docsis_des_dec_ooo
void * docsis_des_dec_ooo
Definition intel-ipsec-mb.h:1148
+
IMB_MGR::gmac192_init
aes_gmac_init_t gmac192_init
Definition intel-ipsec-mb.h:1085
+
IMB_MGR::eia3_1_buffer
zuc_eia3_1_buffer_t eia3_1_buffer
Definition intel-ipsec-mb.h:1052
+
IMB_MGR::aes_ccm_ooo
void * aes_ccm_ooo
Definition intel-ipsec-mb.h:1157
+
IMB_MGR::hec_64
hec_64_t hec_64
Definition intel-ipsec-mb.h:1094
+
IMB_MGR::keyexp_256
keyexp_t keyexp_256
Definition intel-ipsec-mb.h:1004
+
IMB_MGR::aes_ecb_128_quic
aes_ecb_quic_t aes_ecb_128_quic
Definition intel-ipsec-mb.h:1125
+
IMB_MGR::gcm256_pre
aes_gcm_pre_t gcm256_pre
Definition intel-ipsec-mb.h:1047
+
IMB_MGR::used_arch
uint32_t used_arch
Definition intel-ipsec-mb.h:986
+
IMB_MGR::hmac_sha_1_ooo
void * hmac_sha_1_ooo
Definition intel-ipsec-mb.h:1150
+
IMB_MGR::crc32_sctp
crc32_fn_t crc32_sctp
Definition intel-ipsec-mb.h:1099
+
IMB_MGR::crc11_fp_header
crc32_fn_t crc11_fp_header
Definition intel-ipsec-mb.h:1103
+
IMB_MGR::gcm192_dec_finalize
aes_gcm_enc_dec_finalize_t gcm192_dec_finalize
Definition intel-ipsec-mb.h:1040
+
IMB_MGR::f9_1_buffer_user
kasumi_f9_1_buffer_user_t f9_1_buffer_user
Definition intel-ipsec-mb.h:1061
+
IMB_MGR::sha_384_ooo
void * sha_384_ooo
Definition intel-ipsec-mb.h:1171
+
IMB_MGR::gmac192_update
aes_gmac_update_t gmac192_update
Definition intel-ipsec-mb.h:1088
+
IMB_MGR::crc16_x25
crc32_fn_t crc16_x25
Definition intel-ipsec-mb.h:1098
+
IMB_MGR::sha224
hash_fn_t sha224
Definition intel-ipsec-mb.h:1015
+
IMB_MGR::md5_one_block
hash_one_block_t md5_one_block
Definition intel-ipsec-mb.h:1013
+
IMB_MGR::des_enc_ooo
void * des_enc_ooo
Definition intel-ipsec-mb.h:1143
+
IMB_MGR::kasumi_init_f9_key_sched
kasumi_init_f9_key_sched_t kasumi_init_f9_key_sched
Definition intel-ipsec-mb.h:1063
+
IMB_MGR::queue_size
queue_size_t queue_size
Definition intel-ipsec-mb.h:1001
+
IMB_MGR::gcm256_dec_finalize
aes_gcm_enc_dec_finalize_t gcm256_dec_finalize
Definition intel-ipsec-mb.h:1041
+
IMB_MGR::snow3g_key_sched_size
snow3g_key_sched_size_t snow3g_key_sched_size
Definition intel-ipsec-mb.h:1076
+
IMB_MGR::aes_ecb_256_quic
aes_ecb_quic_t aes_ecb_256_quic
Definition intel-ipsec-mb.h:1126
+
IMB_MGR::crc8_wimax_ofdma_hcs
crc32_fn_t crc8_wimax_ofdma_hcs
Definition intel-ipsec-mb.h:1108
+
IMB_MGR::zuc256_eia3_ooo
void * zuc256_eia3_ooo
Definition intel-ipsec-mb.h:1163
+
IMB_MGR::snow3g_f8_8_buffer
snow3g_f8_8_buffer_t snow3g_f8_8_buffer
Definition intel-ipsec-mb.h:1070
+
IMB_MGR::sha512
hash_fn_t sha512
Definition intel-ipsec-mb.h:1018
+
IMB_MGR::hmac_sha_256_ooo
void * hmac_sha_256_ooo
Definition intel-ipsec-mb.h:1152
+
IMB_MGR::snow3g_f8_n_buffer_multikey
snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey
Definition intel-ipsec-mb.h:1073
+
IMB_MGR::gcm192_dec_update
aes_gcm_enc_dec_update_t gcm192_dec_update
Definition intel-ipsec-mb.h:1034
+
IMB_MGR::chacha20_poly1305_init
chacha_poly_init_t chacha20_poly1305_init
Definition intel-ipsec-mb.h:1110
+
IMB_MGR::eea3_n_buffer
zuc_eea3_n_buffer_t eea3_n_buffer
Definition intel-ipsec-mb.h:1051
+
IMB_MGR::gcm128_precomp
aes_gcm_precomp_t gcm128_precomp
Definition intel-ipsec-mb.h:1042
+
IMB_MGR::aes256_ooo
void * aes256_ooo
Definition intel-ipsec-mb.h:1138
+
IMB_MGR::zuc256_eea3_ooo
void * zuc256_eea3_ooo
Definition intel-ipsec-mb.h:1162
+
IMB_MGR::end_ooo
void * end_ooo
Definition intel-ipsec-mb.h:1173
+
IMB_MGR::sha_512_ooo
void * sha_512_ooo
Definition intel-ipsec-mb.h:1172
+
IMB_MGR::chacha20_poly1305_dec_update
chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update
Definition intel-ipsec-mb.h:1112
+
IMB_MGR::gcm256_enc
aes_gcm_enc_dec_t gcm256_enc
Definition intel-ipsec-mb.h:1023
+
IMB_MGR::gcm192_init_var_iv
aes_gcm_init_var_iv_t gcm192_init_var_iv
Definition intel-ipsec-mb.h:1081
+
IMB_MGR::keyexp_128
keyexp_t keyexp_128
Definition intel-ipsec-mb.h:1002
+
IMB_MGR::crc10_iuup_data
crc32_fn_t crc10_iuup_data
Definition intel-ipsec-mb.h:1105
+
IMB_MGR::submit_hash_burst_nocheck
submit_hash_burst_t submit_hash_burst_nocheck
Definition intel-ipsec-mb.h:1122
+
IMB_MGR::sha1_one_block
hash_one_block_t sha1_one_block
Definition intel-ipsec-mb.h:1008
+
IMB_MGR::snow3g_f8_n_buffer
snow3g_f8_n_buffer_t snow3g_f8_n_buffer
Definition intel-ipsec-mb.h:1071
+
IMB_MGR::sha1
hash_fn_t sha1
Definition intel-ipsec-mb.h:1014
+
IMB_MGR::crc32_ethernet_fcs
crc32_fn_t crc32_ethernet_fcs
Definition intel-ipsec-mb.h:1097
+
IMB_MGR::hmac_md5_ooo
void * hmac_md5_ooo
Definition intel-ipsec-mb.h:1155
+
IMB_MGR::des3_enc_ooo
void * des3_enc_ooo
Definition intel-ipsec-mb.h:1145
+
IMB_MGR::sha512_one_block
hash_one_block_t sha512_one_block
Definition intel-ipsec-mb.h:1012
+
IMB_MGR::gcm256_init
aes_gcm_init_t gcm256_init
Definition intel-ipsec-mb.h:1029
+
IMB_MGR::snow3g_f8_8_buffer_multikey
snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey
Definition intel-ipsec-mb.h:1072
+
IMB_MGR::submit_cipher_burst
submit_cipher_burst_t submit_cipher_burst
Definition intel-ipsec-mb.h:1119
+
IMB_MGR::get_next_burst
burst_fn_t get_next_burst
Definition intel-ipsec-mb.h:1115
+
IMB_MGR::sha256
hash_fn_t sha256
Definition intel-ipsec-mb.h:1016
+
IMB_MGR::gcm128_enc_finalize
aes_gcm_enc_dec_finalize_t gcm128_enc_finalize
Definition intel-ipsec-mb.h:1036
+
IMB_MGR::gmac128_update
aes_gmac_update_t gmac128_update
Definition intel-ipsec-mb.h:1087
+
IMB_MGR::zuc_eia3_ooo
void * zuc_eia3_ooo
Definition intel-ipsec-mb.h:1160
+
IMB_MGR::gcm192_enc_finalize
aes_gcm_enc_dec_finalize_t gcm192_enc_finalize
Definition intel-ipsec-mb.h:1037
+
IMB_MGR::f8_3_buffer
kasumi_f8_3_buffer_t f8_3_buffer
Definition intel-ipsec-mb.h:1057
+
IMB_MGR::sha384_one_block
hash_one_block_t sha384_one_block
Definition intel-ipsec-mb.h:1011
+
IMB_MGR::docsis_des_enc_ooo
void * docsis_des_enc_ooo
Definition intel-ipsec-mb.h:1147
+
IMB_MGR::docsis128_sec_ooo
void * docsis128_sec_ooo
Definition intel-ipsec-mb.h:1139
+
IMB_MGR::set_suite_id
void(* set_suite_id)(struct IMB_MGR *, IMB_JOB *)
Definition intel-ipsec-mb.h:1128
+
IMB_MGR::gcm256_enc_finalize
aes_gcm_enc_dec_finalize_t gcm256_enc_finalize
Definition intel-ipsec-mb.h:1038
+
IMB_MGR::aes128_ooo
void * aes128_ooo
Definition intel-ipsec-mb.h:1136
+
IMB_MGR::get_completed_job
get_completed_job_t get_completed_job
Definition intel-ipsec-mb.h:999
+
IMB_MGR::gcm256_dec_update
aes_gcm_enc_dec_update_t gcm256_dec_update
Definition intel-ipsec-mb.h:1035
+
IMB_MGR::snow3g_f9_1_buffer
snow3g_f9_1_buffer_t snow3g_f9_1_buffer
Definition intel-ipsec-mb.h:1074
+
IMB_MGR::aes128_cbcs_ooo
void * aes128_cbcs_ooo
Definition intel-ipsec-mb.h:1161
+
IMB_MGR::gcm128_enc
aes_gcm_enc_dec_t gcm128_enc
Definition intel-ipsec-mb.h:1021
+
IMB_MGR::sha384
hash_fn_t sha384
Definition intel-ipsec-mb.h:1017
+
IMB_MGR::flush_burst
burst_fn_t flush_burst
Definition intel-ipsec-mb.h:1118
+
IMB_MGR::sha_224_ooo
void * sha_224_ooo
Definition intel-ipsec-mb.h:1169
+
IMB_MGR::cmac_subkey_gen_128
cmac_subkey_gen_t cmac_subkey_gen_128
Definition intel-ipsec-mb.h:1005
+
IMB_MGR::crc6_iuup_header
crc32_fn_t crc6_iuup_header
Definition intel-ipsec-mb.h:1106
+
IMB_MGR::submit_burst
burst_fn_t submit_burst
Definition intel-ipsec-mb.h:1116
+
IMB_MGR::snow3g_uia2_ooo
void * snow3g_uia2_ooo
Definition intel-ipsec-mb.h:1167
+
IMB_MGR::gcm192_enc
aes_gcm_enc_dec_t gcm192_enc
Definition intel-ipsec-mb.h:1022
+
IMB_MGR::submit_job_nocheck
submit_job_t submit_job_nocheck
Definition intel-ipsec-mb.h:998
+
IMB_MGR::f8_1_buffer_bit
kasumi_f8_1_buffer_bit_t f8_1_buffer_bit
Definition intel-ipsec-mb.h:1055
+
IMB_MGR::sha224_one_block
hash_one_block_t sha224_one_block
Definition intel-ipsec-mb.h:1009
+
IMB_MGR::des_key_sched
des_keysched_t des_key_sched
Definition intel-ipsec-mb.h:1007
+
IMB_MGR::snow3g_uea2_ooo
void * snow3g_uea2_ooo
Definition intel-ipsec-mb.h:1166
+
IMB_MGR::submit_job
submit_job_t submit_job
Definition intel-ipsec-mb.h:997
+
IMB_MGR::chacha20_poly1305_finalize
chacha_poly_finalize_t chacha20_poly1305_finalize
Definition intel-ipsec-mb.h:1113
+
IMB_MGR::chacha20_poly1305_enc_update
chacha_poly_enc_dec_update_t chacha20_poly1305_enc_update
Definition intel-ipsec-mb.h:1111
+
IMB_MGR::kasumi_init_f8_key_sched
kasumi_init_f8_key_sched_t kasumi_init_f8_key_sched
Definition intel-ipsec-mb.h:1062
+
IMB_MGR::jobs
IMB_JOB jobs[IMB_MAX_JOBS]
Definition intel-ipsec-mb.h:1133
+
IMB_MGR::aes256_cfb_one
aes_cfb_t aes256_cfb_one
Definition intel-ipsec-mb.h:1123
+
IMB_MGR::eia3_n_buffer
zuc_eia3_n_buffer_t eia3_n_buffer
Definition intel-ipsec-mb.h:1079
+
IMB_MGR::snow3g_f8_1_buffer
snow3g_f8_1_buffer_t snow3g_f8_1_buffer
Definition intel-ipsec-mb.h:1067
+
IMB_MGR::hmac_sha_224_ooo
void * hmac_sha_224_ooo
Definition intel-ipsec-mb.h:1151
+
IMB_MGR::kasumi_key_sched_size
kasumi_key_sched_size_t kasumi_key_sched_size
Definition intel-ipsec-mb.h:1064
+
IMB_MGR::aes128_cfb_one
aes_cfb_t aes128_cfb_one
Definition intel-ipsec-mb.h:1019
+
IMB_MGR::cmac_subkey_gen_256
cmac_subkey_gen_t cmac_subkey_gen_256
Definition intel-ipsec-mb.h:1095
+
IMB_MGR::crc24_lte_b
crc32_fn_t crc24_lte_b
Definition intel-ipsec-mb.h:1101
+
IMB_MGR::gcm192_precomp
aes_gcm_precomp_t gcm192_precomp
Definition intel-ipsec-mb.h:1043
+
IMB_MGR::gcm256_enc_update
aes_gcm_enc_dec_update_t gcm256_enc_update
Definition intel-ipsec-mb.h:1032
+
IMB_MGR::eea3_4_buffer
zuc_eea3_4_buffer_t eea3_4_buffer
Definition intel-ipsec-mb.h:1050
+
IMB_MGR::earliest_job
int earliest_job
Definition intel-ipsec-mb.h:1131
+
IMB_MGR::f8_2_buffer
kasumi_f8_2_buffer_t f8_2_buffer
Definition intel-ipsec-mb.h:1056
+
IMB_MGR::crc7_fp_header
crc32_fn_t crc7_fp_header
Definition intel-ipsec-mb.h:1104
+
IMB_MGR::gmac256_init
aes_gmac_init_t gmac256_init
Definition intel-ipsec-mb.h:1086
+
IMB_MGR::submit_cipher_burst_nocheck
submit_cipher_burst_t submit_cipher_burst_nocheck
Definition intel-ipsec-mb.h:1120
+
IMB_MGR::docsis256_sec_ooo
void * docsis256_sec_ooo
Definition intel-ipsec-mb.h:1141
+
IMB_MGR::aes256_cmac_ooo
void * aes256_cmac_ooo
Definition intel-ipsec-mb.h:1165
+
IMB_MGR::sha_1_ooo
void * sha_1_ooo
Definition intel-ipsec-mb.h:1168
+
IMB_MGR::docsis128_crc32_sec_ooo
void * docsis128_crc32_sec_ooo
Definition intel-ipsec-mb.h:1140
+
IMB_MGR::gcm256_dec
aes_gcm_enc_dec_t gcm256_dec
Definition intel-ipsec-mb.h:1026
+
IMB_MGR::crc32_wimax_ofdma_data
crc32_fn_t crc32_wimax_ofdma_data
Definition intel-ipsec-mb.h:1107
+
IMB_MGR::gcm128_enc_update
aes_gcm_enc_dec_update_t gcm128_enc_update
Definition intel-ipsec-mb.h:1030
+
IMB_MGR::aes_xcbc_ooo
void * aes_xcbc_ooo
Definition intel-ipsec-mb.h:1156
+
IMB_MGR::crc16_fp_data
crc32_fn_t crc16_fp_data
Definition intel-ipsec-mb.h:1102
+
IMB_MGR::snow3g_f8_1_buffer_bit
snow3g_f8_1_buffer_bit_t snow3g_f8_1_buffer_bit
Definition intel-ipsec-mb.h:1066
+
IMB_MGR::gcm128_init
aes_gcm_init_t gcm128_init
Definition intel-ipsec-mb.h:1027
+
IMB_MGR::sha256_one_block
hash_one_block_t sha256_one_block
Definition intel-ipsec-mb.h:1010
+
IMB_MGR::gcm192_dec
aes_gcm_enc_dec_t gcm192_dec
Definition intel-ipsec-mb.h:1025
+
IMB_MGR::f8_n_buffer
kasumi_f8_n_buffer_t f8_n_buffer
Definition intel-ipsec-mb.h:1059
+
IMB_MGR::flags
uint64_t flags
Definition intel-ipsec-mb.h:982
+
IMB_MGR::xcbc_keyexp
xcbc_keyexp_t xcbc_keyexp
Definition intel-ipsec-mb.h:1006
+
IMB_MGR::gcm128_init_var_iv
aes_gcm_init_var_iv_t gcm128_init_var_iv
Definition intel-ipsec-mb.h:1080
+
IMB_MGR::eea3_1_buffer
zuc_eea3_1_buffer_t eea3_1_buffer
Definition intel-ipsec-mb.h:1049
+
IMB_MGR::des_dec_ooo
void * des_dec_ooo
Definition intel-ipsec-mb.h:1144
+
IMB_MGR::f8_4_buffer
kasumi_f8_4_buffer_t f8_4_buffer
Definition intel-ipsec-mb.h:1058
+
IMB_MGR::hmac_sha_512_ooo
void * hmac_sha_512_ooo
Definition intel-ipsec-mb.h:1154
+
IMB_MGR::docsis256_crc32_sec_ooo
void * docsis256_crc32_sec_ooo
Definition intel-ipsec-mb.h:1142
+
IMB_MGR::ghash_pre
aes_gcm_pre_t ghash_pre
Definition intel-ipsec-mb.h:1096
+
IMB_MGR::f8_1_buffer
kasumi_f8_1_buffer_t f8_1_buffer
Definition intel-ipsec-mb.h:1054
+
IMB_MGR::features
uint64_t features
Definition intel-ipsec-mb.h:983
+
IMB_MGR::gcm128_dec_finalize
aes_gcm_enc_dec_finalize_t gcm128_dec_finalize
Definition intel-ipsec-mb.h:1039
+
IMB_MGR::gcm256_init_var_iv
aes_gcm_init_var_iv_t gcm256_init_var_iv
Definition intel-ipsec-mb.h:1082
+
IMB_MGR::f9_1_buffer
kasumi_f9_1_buffer_t f9_1_buffer
Definition intel-ipsec-mb.h:1060
+
IMB_MGR::ghash
ghash_t ghash
Definition intel-ipsec-mb.h:1078
+
IMB_MGR::hmac_sha_384_ooo
void * hmac_sha_384_ooo
Definition intel-ipsec-mb.h:1153
+
IMB_MGR::aes192_ooo
void * aes192_ooo
Definition intel-ipsec-mb.h:1137
+
IMB_SGL_IOV
Definition intel-ipsec-mb.h:365
+
IMB_SGL_IOV::len
uint64_t len
Definition intel-ipsec-mb.h:368
+
IMB_SGL_IOV::out
void * out
Definition intel-ipsec-mb.h:367
+
IMB_SGL_IOV::in
const void * in
Definition intel-ipsec-mb.h:366
+
chacha20_poly1305_context_data
holds Chacha20-Poly1305 operation context
Definition intel-ipsec-mb.h:591
+
chacha20_poly1305_context_data::remain_ct_bytes
uint64_t remain_ct_bytes
Definition intel-ipsec-mb.h:601
+
chacha20_poly1305_context_data::aad_len
uint64_t aad_len
Definition intel-ipsec-mb.h:593
+
chacha20_poly1305_context_data::remain_ks_bytes
uint64_t remain_ks_bytes
Definition intel-ipsec-mb.h:599
+
chacha20_poly1305_context_data::poly_scratch
uint8_t poly_scratch[16]
Definition intel-ipsec-mb.h:597
+
chacha20_poly1305_context_data::poly_key
uint8_t poly_key[32]
Definition intel-ipsec-mb.h:596
+
chacha20_poly1305_context_data::IV
uint8_t IV[12]
Definition intel-ipsec-mb.h:604
+
chacha20_poly1305_context_data::last_block_count
uint64_t last_block_count
Definition intel-ipsec-mb.h:598
+
chacha20_poly1305_context_data::hash_len
uint64_t hash_len
Definition intel-ipsec-mb.h:594
+
chacha20_poly1305_context_data::hash
uint64_t hash[3]
Definition intel-ipsec-mb.h:592
+
chacha20_poly1305_context_data::last_ks
uint8_t last_ks[64]
Definition intel-ipsec-mb.h:595
+
gcm_context_data
holds GCM operation context
Definition intel-ipsec-mb.h:577
+
gcm_context_data::in_length
uint64_t in_length
Definition intel-ipsec-mb.h:580
+
gcm_context_data::current_counter
uint8_t current_counter[IMB_GCM_BLOCK_LEN]
Definition intel-ipsec-mb.h:583
+
gcm_context_data::partial_block_enc_key
uint8_t partial_block_enc_key[IMB_GCM_BLOCK_LEN]
Definition intel-ipsec-mb.h:581
+
gcm_context_data::orig_IV
uint8_t orig_IV[IMB_GCM_BLOCK_LEN]
Definition intel-ipsec-mb.h:582
+
gcm_context_data::partial_block_length
uint64_t partial_block_length
Definition intel-ipsec-mb.h:584
+
gcm_context_data::aad_hash
uint8_t aad_hash[IMB_GCM_BLOCK_LEN]
Definition intel-ipsec-mb.h:578
+
gcm_context_data::aad_length
uint64_t aad_length
Definition intel-ipsec-mb.h:579
+
gcm_key_data
holds intermediate key data needed to improve performance
Definition intel-ipsec-mb.h:639
+
gcm_key_data::shifted_hkey
uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN *8]
Definition intel-ipsec-mb.h:649
+
gcm_key_data::avx2_avx512
struct gcm_key_data::@12::@14 avx2_avx512
+
gcm_key_data::shifted_hkey_k
uint8_t shifted_hkey_k[IMB_GCM_ENC_KEY_LEN *8]
Definition intel-ipsec-mb.h:658
+
gcm_key_data::vaes_avx512
struct gcm_key_data::@12::@15 vaes_avx512
+
gcm_key_data::sse_avx
struct gcm_key_data::@12::@13 sse_avx
+
gcm_key_data::expanded_keys
uint8_t expanded_keys[IMB_GCM_ENC_KEY_LEN *IMB_GCM_KEY_SETS]
Definition intel-ipsec-mb.h:640
+
gcm_key_data::ghash_keys
union gcm_key_data::@12 ghash_keys
+
imb_uint128_t
Definition intel-ipsec-mb.h:40
+
imb_uint128_t::low
uint64_t low
Definition intel-ipsec-mb.h:41
+
imb_uint128_t::high
uint64_t high
Definition intel-ipsec-mb.h:42
+
kasumi_key_sched_s
Definition intel-ipsec-mb.h:563
+
kasumi_key_sched_s::msk16
uint16_t msk16[KASUMI_KEY_SCHEDULE_SIZE]
Definition intel-ipsec-mb.h:566
+
kasumi_key_sched_s::sk16
uint16_t sk16[KASUMI_KEY_SCHEDULE_SIZE]
Definition intel-ipsec-mb.h:565
+
snow3g_key_schedule_s
Definition intel-ipsec-mb.h:839
+
snow3g_key_schedule_s::k
uint32_t k[4]
Definition intel-ipsec-mb.h:841
+
+
+ + + + diff --git a/docs/jquery.js b/docs/jquery.js new file mode 100644 index 0000000000000000000000000000000000000000..1dffb65b58c82b5827744a7fd9cd8ce5ab6e4005 --- /dev/null +++ b/docs/jquery.js @@ -0,0 +1,34 @@ +/*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="
",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0",options:{classes:{},disabled:!1,create:null},_createWidget:function(t,e){e=y(e||this.defaultElement||this)[0],this.element=y(e),this.uuid=i++,this.eventNamespace="."+this.widgetName+this.uuid,this.bindings=y(),this.hoverable=y(),this.focusable=y(),this.classesElementLookup={},e!==this&&(y.data(e,this.widgetFullName,this),this._on(!0,this.element,{remove:function(t){t.target===e&&this.destroy()}}),this.document=y(e.style?e.ownerDocument:e.document||e),this.window=y(this.document[0].defaultView||this.document[0].parentWindow)),this.options=y.widget.extend({},this.options,this._getCreateOptions(),t),this._create(),this.options.disabled&&this._setOptionDisabled(this.options.disabled),this._trigger("create",null,this._getCreateEventData()),this._init()},_getCreateOptions:function(){return{}},_getCreateEventData:y.noop,_create:y.noop,_init:y.noop,destroy:function(){var i=this;this._destroy(),y.each(this.classesElementLookup,function(t,e){i._removeClass(e,t)}),this.element.off(this.eventNamespace).removeData(this.widgetFullName),this.widget().off(this.eventNamespace).removeAttr("aria-disabled"),this.bindings.off(this.eventNamespace)},_destroy:y.noop,widget:function(){return this.element},option:function(t,e){var i,s,n,o=t;if(0===arguments.length)return y.widget.extend({},this.options);if("string"==typeof t)if(o={},t=(i=t.split(".")).shift(),i.length){for(s=o[t]=y.widget.extend({},this.options[t]),n=0;n
"),i=e.children()[0];return y("body").append(e),t=i.offsetWidth,e.css("overflow","scroll"),t===(i=i.offsetWidth)&&(i=e[0].clientWidth),e.remove(),s=t-i},getScrollInfo:function(t){var e=t.isWindow||t.isDocument?"":t.element.css("overflow-x"),i=t.isWindow||t.isDocument?"":t.element.css("overflow-y"),e="scroll"===e||"auto"===e&&t.widthx(D(s),D(n))?o.important="horizontal":o.important="vertical",p.using.call(this,t,o)}),h.offset(y.extend(l,{using:t}))})},y.ui.position={fit:{left:function(t,e){var i=e.within,s=i.isWindow?i.scrollLeft:i.offset.left,n=i.width,o=t.left-e.collisionPosition.marginLeft,h=s-o,a=o+e.collisionWidth-n-s;e.collisionWidth>n?0n?0=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return!0}}),y.ui.plugin={add:function(t,e,i){var s,n=y.ui[t].prototype;for(s in i)n.plugins[s]=n.plugins[s]||[],n.plugins[s].push([e,i[s]])},call:function(t,e,i,s){var n,o=t.plugins[e];if(o&&(s||t.element[0].parentNode&&11!==t.element[0].parentNode.nodeType))for(n=0;n").css({overflow:"hidden",position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")})),this.element=this.element.parent().data("ui-resizable",this.element.resizable("instance")),this.elementIsWrapper=!0,t={marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom"),marginLeft:this.originalElement.css("marginLeft")},this.element.css(t),this.originalElement.css("margin",0),this.originalResizeStyle=this.originalElement.css("resize"),this.originalElement.css("resize","none"),this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"})),this.originalElement.css(t),this._proportionallyResize()),this._setupHandles(),e.autoHide&&y(this.element).on("mouseenter",function(){e.disabled||(i._removeClass("ui-resizable-autohide"),i._handles.show())}).on("mouseleave",function(){e.disabled||i.resizing||(i._addClass("ui-resizable-autohide"),i._handles.hide())}),this._mouseInit()},_destroy:function(){this._mouseDestroy(),this._addedHandles.remove();function t(t){y(t).removeData("resizable").removeData("ui-resizable").off(".resizable")}var e;return this.elementIsWrapper&&(t(this.element),e=this.element,this.originalElement.css({position:e.css("position"),width:e.outerWidth(),height:e.outerHeight(),top:e.css("top"),left:e.css("left")}).insertAfter(e),e.remove()),this.originalElement.css("resize",this.originalResizeStyle),t(this.originalElement),this},_setOption:function(t,e){switch(this._super(t,e),t){case"handles":this._removeHandles(),this._setupHandles();break;case"aspectRatio":this._aspectRatio=!!e}},_setupHandles:function(){var t,e,i,s,n,o=this.options,h=this;if(this.handles=o.handles||(y(".ui-resizable-handle",this.element).length?{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"}:"e,s,se"),this._handles=y(),this._addedHandles=y(),this.handles.constructor===String)for("all"===this.handles&&(this.handles="n,e,s,w,se,sw,ne,nw"),i=this.handles.split(","),this.handles={},e=0;e"),this._addClass(n,"ui-resizable-handle "+s),n.css({zIndex:o.zIndex}),this.handles[t]=".ui-resizable-"+t,this.element.children(this.handles[t]).length||(this.element.append(n),this._addedHandles=this._addedHandles.add(n));this._renderAxis=function(t){var e,i,s;for(e in t=t||this.element,this.handles)this.handles[e].constructor===String?this.handles[e]=this.element.children(this.handles[e]).first().show():(this.handles[e].jquery||this.handles[e].nodeType)&&(this.handles[e]=y(this.handles[e]),this._on(this.handles[e],{mousedown:h._mouseDown})),this.elementIsWrapper&&this.originalElement[0].nodeName.match(/^(textarea|input|select|button)$/i)&&(i=y(this.handles[e],this.element),s=/sw|ne|nw|se|n|s/.test(e)?i.outerHeight():i.outerWidth(),i=["padding",/ne|nw|n/.test(e)?"Top":/se|sw|s/.test(e)?"Bottom":/^e$/.test(e)?"Right":"Left"].join(""),t.css(i,s),this._proportionallyResize()),this._handles=this._handles.add(this.handles[e])},this._renderAxis(this.element),this._handles=this._handles.add(this.element.find(".ui-resizable-handle")),this._handles.disableSelection(),this._handles.on("mouseover",function(){h.resizing||(this.className&&(n=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i)),h.axis=n&&n[1]?n[1]:"se")}),o.autoHide&&(this._handles.hide(),this._addClass("ui-resizable-autohide"))},_removeHandles:function(){this._addedHandles.remove()},_mouseCapture:function(t){var e,i,s=!1;for(e in this.handles)(i=y(this.handles[e])[0])!==t.target&&!y.contains(i,t.target)||(s=!0);return!this.options.disabled&&s},_mouseStart:function(t){var e,i,s=this.options,n=this.element;return this.resizing=!0,this._renderProxy(),e=this._num(this.helper.css("left")),i=this._num(this.helper.css("top")),s.containment&&(e+=y(s.containment).scrollLeft()||0,i+=y(s.containment).scrollTop()||0),this.offset=this.helper.offset(),this.position={left:e,top:i},this.size=this._helper?{width:this.helper.width(),height:this.helper.height()}:{width:n.width(),height:n.height()},this.originalSize=this._helper?{width:n.outerWidth(),height:n.outerHeight()}:{width:n.width(),height:n.height()},this.sizeDiff={width:n.outerWidth()-n.width(),height:n.outerHeight()-n.height()},this.originalPosition={left:e,top:i},this.originalMousePosition={left:t.pageX,top:t.pageY},this.aspectRatio="number"==typeof s.aspectRatio?s.aspectRatio:this.originalSize.width/this.originalSize.height||1,s=y(".ui-resizable-"+this.axis).css("cursor"),y("body").css("cursor","auto"===s?this.axis+"-resize":s),this._addClass("ui-resizable-resizing"),this._propagate("start",t),!0},_mouseDrag:function(t){var e=this.originalMousePosition,i=this.axis,s=t.pageX-e.left||0,e=t.pageY-e.top||0,i=this._change[i];return this._updatePrevProperties(),i&&(e=i.apply(this,[t,s,e]),this._updateVirtualBoundaries(t.shiftKey),(this._aspectRatio||t.shiftKey)&&(e=this._updateRatio(e,t)),e=this._respectSize(e,t),this._updateCache(e),this._propagate("resize",t),e=this._applyChanges(),!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize(),y.isEmptyObject(e)||(this._updatePrevProperties(),this._trigger("resize",t,this.ui()),this._applyChanges())),!1},_mouseStop:function(t){this.resizing=!1;var e,i,s,n=this.options,o=this;return this._helper&&(s=(e=(i=this._proportionallyResizeElements).length&&/textarea/i.test(i[0].nodeName))&&this._hasScroll(i[0],"left")?0:o.sizeDiff.height,i=e?0:o.sizeDiff.width,e={width:o.helper.width()-i,height:o.helper.height()-s},i=parseFloat(o.element.css("left"))+(o.position.left-o.originalPosition.left)||null,s=parseFloat(o.element.css("top"))+(o.position.top-o.originalPosition.top)||null,n.animate||this.element.css(y.extend(e,{top:s,left:i})),o.helper.height(o.size.height),o.helper.width(o.size.width),this._helper&&!n.animate&&this._proportionallyResize()),y("body").css("cursor","auto"),this._removeClass("ui-resizable-resizing"),this._propagate("stop",t),this._helper&&this.helper.remove(),!1},_updatePrevProperties:function(){this.prevPosition={top:this.position.top,left:this.position.left},this.prevSize={width:this.size.width,height:this.size.height}},_applyChanges:function(){var t={};return this.position.top!==this.prevPosition.top&&(t.top=this.position.top+"px"),this.position.left!==this.prevPosition.left&&(t.left=this.position.left+"px"),this.size.width!==this.prevSize.width&&(t.width=this.size.width+"px"),this.size.height!==this.prevSize.height&&(t.height=this.size.height+"px"),this.helper.css(t),t},_updateVirtualBoundaries:function(t){var e,i,s=this.options,n={minWidth:this._isNumber(s.minWidth)?s.minWidth:0,maxWidth:this._isNumber(s.maxWidth)?s.maxWidth:1/0,minHeight:this._isNumber(s.minHeight)?s.minHeight:0,maxHeight:this._isNumber(s.maxHeight)?s.maxHeight:1/0};(this._aspectRatio||t)&&(e=n.minHeight*this.aspectRatio,i=n.minWidth/this.aspectRatio,s=n.maxHeight*this.aspectRatio,t=n.maxWidth/this.aspectRatio,e>n.minWidth&&(n.minWidth=e),i>n.minHeight&&(n.minHeight=i),st.width,h=this._isNumber(t.height)&&e.minHeight&&e.minHeight>t.height,a=this.originalPosition.left+this.originalSize.width,r=this.originalPosition.top+this.originalSize.height,l=/sw|nw|w/.test(i),i=/nw|ne|n/.test(i);return o&&(t.width=e.minWidth),h&&(t.height=e.minHeight),s&&(t.width=e.maxWidth),n&&(t.height=e.maxHeight),o&&l&&(t.left=a-e.minWidth),s&&l&&(t.left=a-e.maxWidth),h&&i&&(t.top=r-e.minHeight),n&&i&&(t.top=r-e.maxHeight),t.width||t.height||t.left||!t.top?t.width||t.height||t.top||!t.left||(t.left=null):t.top=null,t},_getPaddingPlusBorderDimensions:function(t){for(var e=0,i=[],s=[t.css("borderTopWidth"),t.css("borderRightWidth"),t.css("borderBottomWidth"),t.css("borderLeftWidth")],n=[t.css("paddingTop"),t.css("paddingRight"),t.css("paddingBottom"),t.css("paddingLeft")];e<4;e++)i[e]=parseFloat(s[e])||0,i[e]+=parseFloat(n[e])||0;return{height:i[0]+i[2],width:i[1]+i[3]}},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var t,e=0,i=this.helper||this.element;e").css({overflow:"hidden"}),this._addClass(this.helper,this._helper),this.helper.css({width:this.element.outerWidth(),height:this.element.outerHeight(),position:"absolute",left:this.elementOffset.left+"px",top:this.elementOffset.top+"px",zIndex:++e.zIndex}),this.helper.appendTo("body").disableSelection()):this.helper=this.element},_change:{e:function(t,e){return{width:this.originalSize.width+e}},w:function(t,e){var i=this.originalSize;return{left:this.originalPosition.left+e,width:i.width-e}},n:function(t,e,i){var s=this.originalSize;return{top:this.originalPosition.top+i,height:s.height-i}},s:function(t,e,i){return{height:this.originalSize.height+i}},se:function(t,e,i){return y.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[t,e,i]))},sw:function(t,e,i){return y.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[t,e,i]))},ne:function(t,e,i){return y.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[t,e,i]))},nw:function(t,e,i){return y.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[t,e,i]))}},_propagate:function(t,e){y.ui.plugin.call(this,t,[e,this.ui()]),"resize"!==t&&this._trigger(t,e,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),y.ui.plugin.add("resizable","animate",{stop:function(e){var i=y(this).resizable("instance"),t=i.options,s=i._proportionallyResizeElements,n=s.length&&/textarea/i.test(s[0].nodeName),o=n&&i._hasScroll(s[0],"left")?0:i.sizeDiff.height,h=n?0:i.sizeDiff.width,n={width:i.size.width-h,height:i.size.height-o},h=parseFloat(i.element.css("left"))+(i.position.left-i.originalPosition.left)||null,o=parseFloat(i.element.css("top"))+(i.position.top-i.originalPosition.top)||null;i.element.animate(y.extend(n,o&&h?{top:o,left:h}:{}),{duration:t.animateDuration,easing:t.animateEasing,step:function(){var t={width:parseFloat(i.element.css("width")),height:parseFloat(i.element.css("height")),top:parseFloat(i.element.css("top")),left:parseFloat(i.element.css("left"))};s&&s.length&&y(s[0]).css({width:t.width,height:t.height}),i._updateCache(t),i._propagate("resize",e)}})}}),y.ui.plugin.add("resizable","containment",{start:function(){var i,s,n=y(this).resizable("instance"),t=n.options,e=n.element,o=t.containment,h=o instanceof y?o.get(0):/parent/.test(o)?e.parent().get(0):o;h&&(n.containerElement=y(h),/document/.test(o)||o===document?(n.containerOffset={left:0,top:0},n.containerPosition={left:0,top:0},n.parentData={element:y(document),left:0,top:0,width:y(document).width(),height:y(document).height()||document.body.parentNode.scrollHeight}):(i=y(h),s=[],y(["Top","Right","Left","Bottom"]).each(function(t,e){s[t]=n._num(i.css("padding"+e))}),n.containerOffset=i.offset(),n.containerPosition=i.position(),n.containerSize={height:i.innerHeight()-s[3],width:i.innerWidth()-s[1]},t=n.containerOffset,e=n.containerSize.height,o=n.containerSize.width,o=n._hasScroll(h,"left")?h.scrollWidth:o,e=n._hasScroll(h)?h.scrollHeight:e,n.parentData={element:h,left:t.left,top:t.top,width:o,height:e}))},resize:function(t){var e=y(this).resizable("instance"),i=e.options,s=e.containerOffset,n=e.position,o=e._aspectRatio||t.shiftKey,h={top:0,left:0},a=e.containerElement,t=!0;a[0]!==document&&/static/.test(a.css("position"))&&(h=s),n.left<(e._helper?s.left:0)&&(e.size.width=e.size.width+(e._helper?e.position.left-s.left:e.position.left-h.left),o&&(e.size.height=e.size.width/e.aspectRatio,t=!1),e.position.left=i.helper?s.left:0),n.top<(e._helper?s.top:0)&&(e.size.height=e.size.height+(e._helper?e.position.top-s.top:e.position.top),o&&(e.size.width=e.size.height*e.aspectRatio,t=!1),e.position.top=e._helper?s.top:0),i=e.containerElement.get(0)===e.element.parent().get(0),n=/relative|absolute/.test(e.containerElement.css("position")),i&&n?(e.offset.left=e.parentData.left+e.position.left,e.offset.top=e.parentData.top+e.position.top):(e.offset.left=e.element.offset().left,e.offset.top=e.element.offset().top),n=Math.abs(e.sizeDiff.width+(e._helper?e.offset.left-h.left:e.offset.left-s.left)),s=Math.abs(e.sizeDiff.height+(e._helper?e.offset.top-h.top:e.offset.top-s.top)),n+e.size.width>=e.parentData.width&&(e.size.width=e.parentData.width-n,o&&(e.size.height=e.size.width/e.aspectRatio,t=!1)),s+e.size.height>=e.parentData.height&&(e.size.height=e.parentData.height-s,o&&(e.size.width=e.size.height*e.aspectRatio,t=!1)),t||(e.position.left=e.prevPosition.left,e.position.top=e.prevPosition.top,e.size.width=e.prevSize.width,e.size.height=e.prevSize.height)},stop:function(){var t=y(this).resizable("instance"),e=t.options,i=t.containerOffset,s=t.containerPosition,n=t.containerElement,o=y(t.helper),h=o.offset(),a=o.outerWidth()-t.sizeDiff.width,o=o.outerHeight()-t.sizeDiff.height;t._helper&&!e.animate&&/relative/.test(n.css("position"))&&y(this).css({left:h.left-s.left-i.left,width:a,height:o}),t._helper&&!e.animate&&/static/.test(n.css("position"))&&y(this).css({left:h.left-s.left-i.left,width:a,height:o})}}),y.ui.plugin.add("resizable","alsoResize",{start:function(){var t=y(this).resizable("instance").options;y(t.alsoResize).each(function(){var t=y(this);t.data("ui-resizable-alsoresize",{width:parseFloat(t.width()),height:parseFloat(t.height()),left:parseFloat(t.css("left")),top:parseFloat(t.css("top"))})})},resize:function(t,i){var e=y(this).resizable("instance"),s=e.options,n=e.originalSize,o=e.originalPosition,h={height:e.size.height-n.height||0,width:e.size.width-n.width||0,top:e.position.top-o.top||0,left:e.position.left-o.left||0};y(s.alsoResize).each(function(){var t=y(this),s=y(this).data("ui-resizable-alsoresize"),n={},e=t.parents(i.originalElement[0]).length?["width","height"]:["width","height","top","left"];y.each(e,function(t,e){var i=(s[e]||0)+(h[e]||0);i&&0<=i&&(n[e]=i||null)}),t.css(n)})},stop:function(){y(this).removeData("ui-resizable-alsoresize")}}),y.ui.plugin.add("resizable","ghost",{start:function(){var t=y(this).resizable("instance"),e=t.size;t.ghost=t.originalElement.clone(),t.ghost.css({opacity:.25,display:"block",position:"relative",height:e.height,width:e.width,margin:0,left:0,top:0}),t._addClass(t.ghost,"ui-resizable-ghost"),!1!==y.uiBackCompat&&"string"==typeof t.options.ghost&&t.ghost.addClass(this.options.ghost),t.ghost.appendTo(t.helper)},resize:function(){var t=y(this).resizable("instance");t.ghost&&t.ghost.css({position:"relative",height:t.size.height,width:t.size.width})},stop:function(){var t=y(this).resizable("instance");t.ghost&&t.helper&&t.helper.get(0).removeChild(t.ghost.get(0))}}),y.ui.plugin.add("resizable","grid",{resize:function(){var t,e=y(this).resizable("instance"),i=e.options,s=e.size,n=e.originalSize,o=e.originalPosition,h=e.axis,a="number"==typeof i.grid?[i.grid,i.grid]:i.grid,r=a[0]||1,l=a[1]||1,u=Math.round((s.width-n.width)/r)*r,p=Math.round((s.height-n.height)/l)*l,d=n.width+u,c=n.height+p,f=i.maxWidth&&i.maxWidthd,s=i.minHeight&&i.minHeight>c;i.grid=a,m&&(d+=r),s&&(c+=l),f&&(d-=r),g&&(c-=l),/^(se|s|e)$/.test(h)?(e.size.width=d,e.size.height=c):/^(ne)$/.test(h)?(e.size.width=d,e.size.height=c,e.position.top=o.top-p):/^(sw)$/.test(h)?(e.size.width=d,e.size.height=c,e.position.left=o.left-u):((c-l<=0||d-r<=0)&&(t=e._getPaddingPlusBorderDimensions(this)),0=f[g]?0:Math.min(f[g],n));!a&&1-1){targetElements.on(evt+EVENT_NAMESPACE,function elementToggle(event){$.powerTip.toggle(this,event)})}else{targetElements.on(evt+EVENT_NAMESPACE,function elementOpen(event){$.powerTip.show(this,event)})}});$.each(options.closeEvents,function(idx,evt){if($.inArray(evt,options.openEvents)<0){targetElements.on(evt+EVENT_NAMESPACE,function elementClose(event){$.powerTip.hide(this,!isMouseEvent(event))})}});targetElements.on("keydown"+EVENT_NAMESPACE,function elementKeyDown(event){if(event.keyCode===27){$.powerTip.hide(this,true)}})}return targetElements};$.fn.powerTip.defaults={fadeInTime:200,fadeOutTime:100,followMouse:false,popupId:"powerTip",popupClass:null,intentSensitivity:7,intentPollInterval:100,closeDelay:100,placement:"n",smartPlacement:false,offset:10,mouseOnToPopup:false,manual:false,openEvents:["mouseenter","focus"],closeEvents:["mouseleave","blur"]};$.fn.powerTip.smartPlacementLists={n:["n","ne","nw","s"],e:["e","ne","se","w","nw","sw","n","s","e"],s:["s","se","sw","n"],w:["w","nw","sw","e","ne","se","n","s","w"],nw:["nw","w","sw","n","s","se","nw"],ne:["ne","e","se","n","s","sw","ne"],sw:["sw","w","nw","s","n","ne","sw"],se:["se","e","ne","s","n","nw","se"],"nw-alt":["nw-alt","n","ne-alt","sw-alt","s","se-alt","w","e"],"ne-alt":["ne-alt","n","nw-alt","se-alt","s","sw-alt","e","w"],"sw-alt":["sw-alt","s","se-alt","nw-alt","n","ne-alt","w","e"],"se-alt":["se-alt","s","sw-alt","ne-alt","n","nw-alt","e","w"]};$.powerTip={show:function apiShowTip(element,event){if(isMouseEvent(event)){trackMouse(event);session.previousX=event.pageX;session.previousY=event.pageY;$(element).data(DATA_DISPLAYCONTROLLER).show()}else{$(element).first().data(DATA_DISPLAYCONTROLLER).show(true,true)}return element},reposition:function apiResetPosition(element){$(element).first().data(DATA_DISPLAYCONTROLLER).resetPosition();return element},hide:function apiCloseTip(element,immediate){var displayController;immediate=element?immediate:true;if(element){displayController=$(element).first().data(DATA_DISPLAYCONTROLLER)}else if(session.activeHover){displayController=session.activeHover.data(DATA_DISPLAYCONTROLLER)}if(displayController){displayController.hide(immediate)}return element},toggle:function apiToggle(element,event){if(session.activeHover&&session.activeHover.is(element)){$.powerTip.hide(element,!isMouseEvent(event))}else{$.powerTip.show(element,event)}return element}};$.powerTip.showTip=$.powerTip.show;$.powerTip.closeTip=$.powerTip.hide;function CSSCoordinates(){var me=this;me.top="auto";me.left="auto";me.right="auto";me.bottom="auto";me.set=function(property,value){if($.isNumeric(value)){me[property]=Math.round(value)}}}function DisplayController(element,options,tipController){var hoverTimer=null,myCloseDelay=null;function openTooltip(immediate,forceOpen){cancelTimer();if(!element.data(DATA_HASACTIVEHOVER)){if(!immediate){session.tipOpenImminent=true;hoverTimer=setTimeout(function intentDelay(){hoverTimer=null;checkForIntent()},options.intentPollInterval)}else{if(forceOpen){element.data(DATA_FORCEDOPEN,true)}closeAnyDelayed();tipController.showTip(element)}}else{cancelClose()}}function closeTooltip(disableDelay){if(myCloseDelay){myCloseDelay=session.closeDelayTimeout=clearTimeout(myCloseDelay);session.delayInProgress=false}cancelTimer();session.tipOpenImminent=false;if(element.data(DATA_HASACTIVEHOVER)){element.data(DATA_FORCEDOPEN,false);if(!disableDelay){session.delayInProgress=true;session.closeDelayTimeout=setTimeout(function closeDelay(){session.closeDelayTimeout=null;tipController.hideTip(element);session.delayInProgress=false;myCloseDelay=null},options.closeDelay);myCloseDelay=session.closeDelayTimeout}else{tipController.hideTip(element)}}}function checkForIntent(){var xDifference=Math.abs(session.previousX-session.currentX),yDifference=Math.abs(session.previousY-session.currentY),totalDifference=xDifference+yDifference;if(totalDifference",{id:options.popupId});if($body.length===0){$body=$("body")}$body.append(tipElement);session.tooltips=session.tooltips?session.tooltips.add(tipElement):tipElement}if(options.followMouse){if(!tipElement.data(DATA_HASMOUSEMOVE)){$document.on("mousemove"+EVENT_NAMESPACE,positionTipOnCursor);$window.on("scroll"+EVENT_NAMESPACE,positionTipOnCursor);tipElement.data(DATA_HASMOUSEMOVE,true)}}function beginShowTip(element){element.data(DATA_HASACTIVEHOVER,true);tipElement.queue(function queueTipInit(next){showTip(element);next()})}function showTip(element){var tipContent;if(!element.data(DATA_HASACTIVEHOVER)){return}if(session.isTipOpen){if(!session.isClosing){hideTip(session.activeHover)}tipElement.delay(100).queue(function queueTipAgain(next){showTip(element);next()});return}element.trigger("powerTipPreRender");tipContent=getTooltipContent(element);if(tipContent){tipElement.empty().append(tipContent)}else{return}element.trigger("powerTipRender");session.activeHover=element;session.isTipOpen=true;tipElement.data(DATA_MOUSEONTOTIP,options.mouseOnToPopup);tipElement.addClass(options.popupClass);if(!options.followMouse||element.data(DATA_FORCEDOPEN)){positionTipOnElement(element);session.isFixedTipOpen=true}else{positionTipOnCursor()}if(!element.data(DATA_FORCEDOPEN)&&!options.followMouse){$document.on("click"+EVENT_NAMESPACE,function documentClick(event){var target=event.target;if(target!==element[0]){if(options.mouseOnToPopup){if(target!==tipElement[0]&&!$.contains(tipElement[0],target)){$.powerTip.hide()}}else{$.powerTip.hide()}}})}if(options.mouseOnToPopup&&!options.manual){tipElement.on("mouseenter"+EVENT_NAMESPACE,function tipMouseEnter(){if(session.activeHover){session.activeHover.data(DATA_DISPLAYCONTROLLER).cancel()}});tipElement.on("mouseleave"+EVENT_NAMESPACE,function tipMouseLeave(){if(session.activeHover){session.activeHover.data(DATA_DISPLAYCONTROLLER).hide()}})}tipElement.fadeIn(options.fadeInTime,function fadeInCallback(){if(!session.desyncTimeout){session.desyncTimeout=setInterval(closeDesyncedTip,500)}element.trigger("powerTipOpen")})}function hideTip(element){session.isClosing=true;session.isTipOpen=false;session.desyncTimeout=clearInterval(session.desyncTimeout);element.data(DATA_HASACTIVEHOVER,false);element.data(DATA_FORCEDOPEN,false);$document.off("click"+EVENT_NAMESPACE);tipElement.off(EVENT_NAMESPACE);tipElement.fadeOut(options.fadeOutTime,function fadeOutCallback(){var coords=new CSSCoordinates;session.activeHover=null;session.isClosing=false;session.isFixedTipOpen=false;tipElement.removeClass();coords.set("top",session.currentY+options.offset);coords.set("left",session.currentX+options.offset);tipElement.css(coords);element.trigger("powerTipClose")})}function positionTipOnCursor(){var tipWidth,tipHeight,coords,collisions,collisionCount;if(!session.isFixedTipOpen&&(session.isTipOpen||session.tipOpenImminent&&tipElement.data(DATA_HASMOUSEMOVE))){tipWidth=tipElement.outerWidth();tipHeight=tipElement.outerHeight();coords=new CSSCoordinates;coords.set("top",session.currentY+options.offset);coords.set("left",session.currentX+options.offset);collisions=getViewportCollisions(coords,tipWidth,tipHeight);if(collisions!==Collision.none){collisionCount=countFlags(collisions);if(collisionCount===1){if(collisions===Collision.right){coords.set("left",session.scrollLeft+session.windowWidth-tipWidth)}else if(collisions===Collision.bottom){coords.set("top",session.scrollTop+session.windowHeight-tipHeight)}}else{coords.set("left",session.currentX-tipWidth-options.offset);coords.set("top",session.currentY-tipHeight-options.offset)}}tipElement.css(coords)}}function positionTipOnElement(element){var priorityList,finalPlacement;if(options.smartPlacement||options.followMouse&&element.data(DATA_FORCEDOPEN)){priorityList=$.fn.powerTip.smartPlacementLists[options.placement];$.each(priorityList,function(idx,pos){var collisions=getViewportCollisions(placeTooltip(element,pos),tipElement.outerWidth(),tipElement.outerHeight());finalPlacement=pos;return collisions!==Collision.none})}else{placeTooltip(element,options.placement);finalPlacement=options.placement}tipElement.removeClass("w nw sw e ne se n s w se-alt sw-alt ne-alt nw-alt");tipElement.addClass(finalPlacement)}function placeTooltip(element,placement){var iterationCount=0,tipWidth,tipHeight,coords=new CSSCoordinates;coords.set("top",0);coords.set("left",0);tipElement.css(coords);do{tipWidth=tipElement.outerWidth();tipHeight=tipElement.outerHeight();coords=placementCalculator.compute(element,placement,tipWidth,tipHeight,options.offset);tipElement.css(coords)}while(++iterationCount<=5&&(tipWidth!==tipElement.outerWidth()||tipHeight!==tipElement.outerHeight()));return coords}function closeDesyncedTip(){var isDesynced=false,hasDesyncableCloseEvent=$.grep(["mouseleave","mouseout","blur","focusout"],function(eventType){return $.inArray(eventType,options.closeEvents)!==-1}).length>0;if(session.isTipOpen&&!session.isClosing&&!session.delayInProgress&&hasDesyncableCloseEvent){if(session.activeHover.data(DATA_HASACTIVEHOVER)===false||session.activeHover.is(":disabled")){isDesynced=true}else if(!isMouseOver(session.activeHover)&&!session.activeHover.is(":focus")&&!session.activeHover.data(DATA_FORCEDOPEN)){if(tipElement.data(DATA_MOUSEONTOTIP)){if(!isMouseOver(tipElement)){isDesynced=true}}else{isDesynced=true}}if(isDesynced){hideTip(session.activeHover)}}}this.showTip=beginShowTip;this.hideTip=hideTip;this.resetPosition=positionTipOnElement}function isSvgElement(element){return Boolean(window.SVGElement&&element[0]instanceof SVGElement)}function isMouseEvent(event){return Boolean(event&&$.inArray(event.type,MOUSE_EVENTS)>-1&&typeof event.pageX==="number")}function initTracking(){if(!session.mouseTrackingActive){session.mouseTrackingActive=true;getViewportDimensions();$(getViewportDimensions);$document.on("mousemove"+EVENT_NAMESPACE,trackMouse);$window.on("resize"+EVENT_NAMESPACE,trackResize);$window.on("scroll"+EVENT_NAMESPACE,trackScroll)}}function getViewportDimensions(){session.scrollLeft=$window.scrollLeft();session.scrollTop=$window.scrollTop();session.windowWidth=$window.width();session.windowHeight=$window.height()}function trackResize(){session.windowWidth=$window.width();session.windowHeight=$window.height()}function trackScroll(){var x=$window.scrollLeft(),y=$window.scrollTop();if(x!==session.scrollLeft){session.currentX+=x-session.scrollLeft;session.scrollLeft=x}if(y!==session.scrollTop){session.currentY+=y-session.scrollTop;session.scrollTop=y}}function trackMouse(event){session.currentX=event.pageX;session.currentY=event.pageY}function isMouseOver(element){var elementPosition=element.offset(),elementBox=element[0].getBoundingClientRect(),elementWidth=elementBox.right-elementBox.left,elementHeight=elementBox.bottom-elementBox.top;return session.currentX>=elementPosition.left&&session.currentX<=elementPosition.left+elementWidth&&session.currentY>=elementPosition.top&&session.currentY<=elementPosition.top+elementHeight}function getTooltipContent(element){var tipText=element.data(DATA_POWERTIP),tipObject=element.data(DATA_POWERTIPJQ),tipTarget=element.data(DATA_POWERTIPTARGET),targetElement,content;if(tipText){if($.isFunction(tipText)){tipText=tipText.call(element[0])}content=tipText}else if(tipObject){if($.isFunction(tipObject)){tipObject=tipObject.call(element[0])}if(tipObject.length>0){content=tipObject.clone(true,true)}}else if(tipTarget){targetElement=$("#"+tipTarget);if(targetElement.length>0){content=targetElement.html()}}return content}function getViewportCollisions(coords,elementWidth,elementHeight){var viewportTop=session.scrollTop,viewportLeft=session.scrollLeft,viewportBottom=viewportTop+session.windowHeight,viewportRight=viewportLeft+session.windowWidth,collisions=Collision.none;if(coords.topviewportBottom||Math.abs(coords.bottom-session.windowHeight)>viewportBottom){collisions|=Collision.bottom}if(coords.leftviewportRight){collisions|=Collision.left}if(coords.left+elementWidth>viewportRight||coords.right1)){a.preventDefault();var c=a.originalEvent.changedTouches[0],d=document.createEvent("MouseEvents");d.initMouseEvent(b,!0,!0,window,1,c.screenX,c.screenY,c.clientX,c.clientY,!1,!1,!1,!1,0,null),a.target.dispatchEvent(d)}}if(a.support.touch="ontouchend"in document,a.support.touch){var e,b=a.ui.mouse.prototype,c=b._mouseInit,d=b._mouseDestroy;b._touchStart=function(a){var b=this;!e&&b._mouseCapture(a.originalEvent.changedTouches[0])&&(e=!0,b._touchMoved=!1,f(a,"mouseover"),f(a,"mousemove"),f(a,"mousedown"))},b._touchMove=function(a){e&&(this._touchMoved=!0,f(a,"mousemove"))},b._touchEnd=function(a){e&&(f(a,"mouseup"),f(a,"mouseout"),this._touchMoved||f(a,"click"),e=!1)},b._mouseInit=function(){var b=this;b.element.bind({touchstart:a.proxy(b,"_touchStart"),touchmove:a.proxy(b,"_touchMove"),touchend:a.proxy(b,"_touchEnd")}),c.call(b)},b._mouseDestroy=function(){var b=this;b.element.unbind({touchstart:a.proxy(b,"_touchStart"),touchmove:a.proxy(b,"_touchMove"),touchend:a.proxy(b,"_touchEnd")}),d.call(b)}}}(jQuery);/*! SmartMenus jQuery Plugin - v1.1.0 - September 17, 2017 + * http://www.smartmenus.org/ + * Copyright Vasil Dinkov, Vadikom Web Ltd. http://vadikom.com; Licensed MIT */(function(t){"function"==typeof define&&define.amd?define(["jquery"],t):"object"==typeof module&&"object"==typeof module.exports?module.exports=t(require("jquery")):t(jQuery)})(function($){function initMouseDetection(t){var e=".smartmenus_mouse";if(mouseDetectionEnabled||t)mouseDetectionEnabled&&t&&($(document).off(e),mouseDetectionEnabled=!1);else{var i=!0,s=null,o={mousemove:function(t){var e={x:t.pageX,y:t.pageY,timeStamp:(new Date).getTime()};if(s){var o=Math.abs(s.x-e.x),a=Math.abs(s.y-e.y);if((o>0||a>0)&&2>=o&&2>=a&&300>=e.timeStamp-s.timeStamp&&(mouse=!0,i)){var n=$(t.target).closest("a");n.is("a")&&$.each(menuTrees,function(){return $.contains(this.$root[0],n[0])?(this.itemEnter({currentTarget:n[0]}),!1):void 0}),i=!1}}s=e}};o[touchEvents?"touchstart":"pointerover pointermove pointerout MSPointerOver MSPointerMove MSPointerOut"]=function(t){isTouchEvent(t.originalEvent)&&(mouse=!1)},$(document).on(getEventsNS(o,e)),mouseDetectionEnabled=!0}}function isTouchEvent(t){return!/^(4|mouse)$/.test(t.pointerType)}function getEventsNS(t,e){e||(e="");var i={};for(var s in t)i[s.split(" ").join(e+" ")+e]=t[s];return i}var menuTrees=[],mouse=!1,touchEvents="ontouchstart"in window,mouseDetectionEnabled=!1,requestAnimationFrame=window.requestAnimationFrame||function(t){return setTimeout(t,1e3/60)},cancelAnimationFrame=window.cancelAnimationFrame||function(t){clearTimeout(t)},canAnimate=!!$.fn.animate;return $.SmartMenus=function(t,e){this.$root=$(t),this.opts=e,this.rootId="",this.accessIdPrefix="",this.$subArrow=null,this.activatedItems=[],this.visibleSubMenus=[],this.showTimeout=0,this.hideTimeout=0,this.scrollTimeout=0,this.clickActivated=!1,this.focusActivated=!1,this.zIndexInc=0,this.idInc=0,this.$firstLink=null,this.$firstSub=null,this.disabled=!1,this.$disableOverlay=null,this.$touchScrollingSub=null,this.cssTransforms3d="perspective"in t.style||"webkitPerspective"in t.style,this.wasCollapsible=!1,this.init()},$.extend($.SmartMenus,{hideAll:function(){$.each(menuTrees,function(){this.menuHideAll()})},destroy:function(){for(;menuTrees.length;)menuTrees[0].destroy();initMouseDetection(!0)},prototype:{init:function(t){var e=this;if(!t){menuTrees.push(this),this.rootId=((new Date).getTime()+Math.random()+"").replace(/\D/g,""),this.accessIdPrefix="sm-"+this.rootId+"-",this.$root.hasClass("sm-rtl")&&(this.opts.rightToLeftSubMenus=!0);var i=".smartmenus";this.$root.data("smartmenus",this).attr("data-smartmenus-id",this.rootId).dataSM("level",1).on(getEventsNS({"mouseover focusin":$.proxy(this.rootOver,this),"mouseout focusout":$.proxy(this.rootOut,this),keydown:$.proxy(this.rootKeyDown,this)},i)).on(getEventsNS({mouseenter:$.proxy(this.itemEnter,this),mouseleave:$.proxy(this.itemLeave,this),mousedown:$.proxy(this.itemDown,this),focus:$.proxy(this.itemFocus,this),blur:$.proxy(this.itemBlur,this),click:$.proxy(this.itemClick,this)},i),"a"),i+=this.rootId,this.opts.hideOnClick&&$(document).on(getEventsNS({touchstart:$.proxy(this.docTouchStart,this),touchmove:$.proxy(this.docTouchMove,this),touchend:$.proxy(this.docTouchEnd,this),click:$.proxy(this.docClick,this)},i)),$(window).on(getEventsNS({"resize orientationchange":$.proxy(this.winResize,this)},i)),this.opts.subIndicators&&(this.$subArrow=$("").addClass("sub-arrow"),this.opts.subIndicatorsText&&this.$subArrow.html(this.opts.subIndicatorsText)),initMouseDetection()}if(this.$firstSub=this.$root.find("ul").each(function(){e.menuInit($(this))}).eq(0),this.$firstLink=this.$root.find("a").eq(0),this.opts.markCurrentItem){var s=/(index|default)\.[^#\?\/]*/i,o=/#.*/,a=window.location.href.replace(s,""),n=a.replace(o,"");this.$root.find("a").each(function(){var t=this.href.replace(s,""),i=$(this);(t==a||t==n)&&(i.addClass("current"),e.opts.markCurrentTree&&i.parentsUntil("[data-smartmenus-id]","ul").each(function(){$(this).dataSM("parent-a").addClass("current")}))})}this.wasCollapsible=this.isCollapsible()},destroy:function(t){if(!t){var e=".smartmenus";this.$root.removeData("smartmenus").removeAttr("data-smartmenus-id").removeDataSM("level").off(e),e+=this.rootId,$(document).off(e),$(window).off(e),this.opts.subIndicators&&(this.$subArrow=null)}this.menuHideAll();var i=this;this.$root.find("ul").each(function(){var t=$(this);t.dataSM("scroll-arrows")&&t.dataSM("scroll-arrows").remove(),t.dataSM("shown-before")&&((i.opts.subMenusMinWidth||i.opts.subMenusMaxWidth)&&t.css({width:"",minWidth:"",maxWidth:""}).removeClass("sm-nowrap"),t.dataSM("scroll-arrows")&&t.dataSM("scroll-arrows").remove(),t.css({zIndex:"",top:"",left:"",marginLeft:"",marginTop:"",display:""})),0==(t.attr("id")||"").indexOf(i.accessIdPrefix)&&t.removeAttr("id")}).removeDataSM("in-mega").removeDataSM("shown-before").removeDataSM("scroll-arrows").removeDataSM("parent-a").removeDataSM("level").removeDataSM("beforefirstshowfired").removeAttr("role").removeAttr("aria-hidden").removeAttr("aria-labelledby").removeAttr("aria-expanded"),this.$root.find("a.has-submenu").each(function(){var t=$(this);0==t.attr("id").indexOf(i.accessIdPrefix)&&t.removeAttr("id")}).removeClass("has-submenu").removeDataSM("sub").removeAttr("aria-haspopup").removeAttr("aria-controls").removeAttr("aria-expanded").closest("li").removeDataSM("sub"),this.opts.subIndicators&&this.$root.find("span.sub-arrow").remove(),this.opts.markCurrentItem&&this.$root.find("a.current").removeClass("current"),t||(this.$root=null,this.$firstLink=null,this.$firstSub=null,this.$disableOverlay&&(this.$disableOverlay.remove(),this.$disableOverlay=null),menuTrees.splice($.inArray(this,menuTrees),1))},disable:function(t){if(!this.disabled){if(this.menuHideAll(),!t&&!this.opts.isPopup&&this.$root.is(":visible")){var e=this.$root.offset();this.$disableOverlay=$('
').css({position:"absolute",top:e.top,left:e.left,width:this.$root.outerWidth(),height:this.$root.outerHeight(),zIndex:this.getStartZIndex(!0),opacity:0}).appendTo(document.body)}this.disabled=!0}},docClick:function(t){return this.$touchScrollingSub?(this.$touchScrollingSub=null,void 0):((this.visibleSubMenus.length&&!$.contains(this.$root[0],t.target)||$(t.target).closest("a").length)&&this.menuHideAll(),void 0)},docTouchEnd:function(){if(this.lastTouch){if(!(!this.visibleSubMenus.length||void 0!==this.lastTouch.x2&&this.lastTouch.x1!=this.lastTouch.x2||void 0!==this.lastTouch.y2&&this.lastTouch.y1!=this.lastTouch.y2||this.lastTouch.target&&$.contains(this.$root[0],this.lastTouch.target))){this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0);var t=this;this.hideTimeout=setTimeout(function(){t.menuHideAll()},350)}this.lastTouch=null}},docTouchMove:function(t){if(this.lastTouch){var e=t.originalEvent.touches[0];this.lastTouch.x2=e.pageX,this.lastTouch.y2=e.pageY}},docTouchStart:function(t){var e=t.originalEvent.touches[0];this.lastTouch={x1:e.pageX,y1:e.pageY,target:e.target}},enable:function(){this.disabled&&(this.$disableOverlay&&(this.$disableOverlay.remove(),this.$disableOverlay=null),this.disabled=!1)},getClosestMenu:function(t){for(var e=$(t).closest("ul");e.dataSM("in-mega");)e=e.parent().closest("ul");return e[0]||null},getHeight:function(t){return this.getOffset(t,!0)},getOffset:function(t,e){var i;"none"==t.css("display")&&(i={position:t[0].style.position,visibility:t[0].style.visibility},t.css({position:"absolute",visibility:"hidden"}).show());var s=t[0].getBoundingClientRect&&t[0].getBoundingClientRect(),o=s&&(e?s.height||s.bottom-s.top:s.width||s.right-s.left);return o||0===o||(o=e?t[0].offsetHeight:t[0].offsetWidth),i&&t.hide().css(i),o},getStartZIndex:function(t){var e=parseInt(this[t?"$root":"$firstSub"].css("z-index"));return!t&&isNaN(e)&&(e=parseInt(this.$root.css("z-index"))),isNaN(e)?1:e},getTouchPoint:function(t){return t.touches&&t.touches[0]||t.changedTouches&&t.changedTouches[0]||t},getViewport:function(t){var e=t?"Height":"Width",i=document.documentElement["client"+e],s=window["inner"+e];return s&&(i=Math.min(i,s)),i},getViewportHeight:function(){return this.getViewport(!0)},getViewportWidth:function(){return this.getViewport()},getWidth:function(t){return this.getOffset(t)},handleEvents:function(){return!this.disabled&&this.isCSSOn()},handleItemEvents:function(t){return this.handleEvents()&&!this.isLinkInMegaMenu(t)},isCollapsible:function(){return"static"==this.$firstSub.css("position")},isCSSOn:function(){return"inline"!=this.$firstLink.css("display")},isFixed:function(){var t="fixed"==this.$root.css("position");return t||this.$root.parentsUntil("body").each(function(){return"fixed"==$(this).css("position")?(t=!0,!1):void 0}),t},isLinkInMegaMenu:function(t){return $(this.getClosestMenu(t[0])).hasClass("mega-menu")},isTouchMode:function(){return!mouse||this.opts.noMouseOver||this.isCollapsible()},itemActivate:function(t,e){var i=t.closest("ul"),s=i.dataSM("level");if(s>1&&(!this.activatedItems[s-2]||this.activatedItems[s-2][0]!=i.dataSM("parent-a")[0])){var o=this;$(i.parentsUntil("[data-smartmenus-id]","ul").get().reverse()).add(i).each(function(){o.itemActivate($(this).dataSM("parent-a"))})}if((!this.isCollapsible()||e)&&this.menuHideSubMenus(this.activatedItems[s-1]&&this.activatedItems[s-1][0]==t[0]?s:s-1),this.activatedItems[s-1]=t,this.$root.triggerHandler("activate.smapi",t[0])!==!1){var a=t.dataSM("sub");a&&(this.isTouchMode()||!this.opts.showOnClick||this.clickActivated)&&this.menuShow(a)}},itemBlur:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&this.$root.triggerHandler("blur.smapi",e[0])},itemClick:function(t){var e=$(t.currentTarget);if(this.handleItemEvents(e)){if(this.$touchScrollingSub&&this.$touchScrollingSub[0]==e.closest("ul")[0])return this.$touchScrollingSub=null,t.stopPropagation(),!1;if(this.$root.triggerHandler("click.smapi",e[0])===!1)return!1;var i=$(t.target).is(".sub-arrow"),s=e.dataSM("sub"),o=s?2==s.dataSM("level"):!1,a=this.isCollapsible(),n=/toggle$/.test(this.opts.collapsibleBehavior),r=/link$/.test(this.opts.collapsibleBehavior),h=/^accordion/.test(this.opts.collapsibleBehavior);if(s&&!s.is(":visible")){if((!r||!a||i)&&(this.opts.showOnClick&&o&&(this.clickActivated=!0),this.itemActivate(e,h),s.is(":visible")))return this.focusActivated=!0,!1}else if(a&&(n||i))return this.itemActivate(e,h),this.menuHide(s),n&&(this.focusActivated=!1),!1;return this.opts.showOnClick&&o||e.hasClass("disabled")||this.$root.triggerHandler("select.smapi",e[0])===!1?!1:void 0}},itemDown:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&e.dataSM("mousedown",!0)},itemEnter:function(t){var e=$(t.currentTarget);if(this.handleItemEvents(e)){if(!this.isTouchMode()){this.showTimeout&&(clearTimeout(this.showTimeout),this.showTimeout=0);var i=this;this.showTimeout=setTimeout(function(){i.itemActivate(e)},this.opts.showOnClick&&1==e.closest("ul").dataSM("level")?1:this.opts.showTimeout)}this.$root.triggerHandler("mouseenter.smapi",e[0])}},itemFocus:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&(!this.focusActivated||this.isTouchMode()&&e.dataSM("mousedown")||this.activatedItems.length&&this.activatedItems[this.activatedItems.length-1][0]==e[0]||this.itemActivate(e,!0),this.$root.triggerHandler("focus.smapi",e[0]))},itemLeave:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&(this.isTouchMode()||(e[0].blur(),this.showTimeout&&(clearTimeout(this.showTimeout),this.showTimeout=0)),e.removeDataSM("mousedown"),this.$root.triggerHandler("mouseleave.smapi",e[0]))},menuHide:function(t){if(this.$root.triggerHandler("beforehide.smapi",t[0])!==!1&&(canAnimate&&t.stop(!0,!0),"none"!=t.css("display"))){var e=function(){t.css("z-index","")};this.isCollapsible()?canAnimate&&this.opts.collapsibleHideFunction?this.opts.collapsibleHideFunction.call(this,t,e):t.hide(this.opts.collapsibleHideDuration,e):canAnimate&&this.opts.hideFunction?this.opts.hideFunction.call(this,t,e):t.hide(this.opts.hideDuration,e),t.dataSM("scroll")&&(this.menuScrollStop(t),t.css({"touch-action":"","-ms-touch-action":"","-webkit-transform":"",transform:""}).off(".smartmenus_scroll").removeDataSM("scroll").dataSM("scroll-arrows").hide()),t.dataSM("parent-a").removeClass("highlighted").attr("aria-expanded","false"),t.attr({"aria-expanded":"false","aria-hidden":"true"});var i=t.dataSM("level");this.activatedItems.splice(i-1,1),this.visibleSubMenus.splice($.inArray(t,this.visibleSubMenus),1),this.$root.triggerHandler("hide.smapi",t[0])}},menuHideAll:function(){this.showTimeout&&(clearTimeout(this.showTimeout),this.showTimeout=0);for(var t=this.opts.isPopup?1:0,e=this.visibleSubMenus.length-1;e>=t;e--)this.menuHide(this.visibleSubMenus[e]);this.opts.isPopup&&(canAnimate&&this.$root.stop(!0,!0),this.$root.is(":visible")&&(canAnimate&&this.opts.hideFunction?this.opts.hideFunction.call(this,this.$root):this.$root.hide(this.opts.hideDuration))),this.activatedItems=[],this.visibleSubMenus=[],this.clickActivated=!1,this.focusActivated=!1,this.zIndexInc=0,this.$root.triggerHandler("hideAll.smapi")},menuHideSubMenus:function(t){for(var e=this.activatedItems.length-1;e>=t;e--){var i=this.activatedItems[e].dataSM("sub");i&&this.menuHide(i)}},menuInit:function(t){if(!t.dataSM("in-mega")){t.hasClass("mega-menu")&&t.find("ul").dataSM("in-mega",!0);for(var e=2,i=t[0];(i=i.parentNode.parentNode)!=this.$root[0];)e++;var s=t.prevAll("a").eq(-1);s.length||(s=t.prevAll().find("a").eq(-1)),s.addClass("has-submenu").dataSM("sub",t),t.dataSM("parent-a",s).dataSM("level",e).parent().dataSM("sub",t);var o=s.attr("id")||this.accessIdPrefix+ ++this.idInc,a=t.attr("id")||this.accessIdPrefix+ ++this.idInc;s.attr({id:o,"aria-haspopup":"true","aria-controls":a,"aria-expanded":"false"}),t.attr({id:a,role:"group","aria-hidden":"true","aria-labelledby":o,"aria-expanded":"false"}),this.opts.subIndicators&&s[this.opts.subIndicatorsPos](this.$subArrow.clone())}},menuPosition:function(t){var e,i,s=t.dataSM("parent-a"),o=s.closest("li"),a=o.parent(),n=t.dataSM("level"),r=this.getWidth(t),h=this.getHeight(t),u=s.offset(),l=u.left,c=u.top,d=this.getWidth(s),m=this.getHeight(s),p=$(window),f=p.scrollLeft(),v=p.scrollTop(),b=this.getViewportWidth(),S=this.getViewportHeight(),g=a.parent().is("[data-sm-horizontal-sub]")||2==n&&!a.hasClass("sm-vertical"),M=this.opts.rightToLeftSubMenus&&!o.is("[data-sm-reverse]")||!this.opts.rightToLeftSubMenus&&o.is("[data-sm-reverse]"),w=2==n?this.opts.mainMenuSubOffsetX:this.opts.subMenusSubOffsetX,T=2==n?this.opts.mainMenuSubOffsetY:this.opts.subMenusSubOffsetY;if(g?(e=M?d-r-w:w,i=this.opts.bottomToTopSubMenus?-h-T:m+T):(e=M?w-r:d-w,i=this.opts.bottomToTopSubMenus?m-T-h:T),this.opts.keepInViewport){var y=l+e,I=c+i;if(M&&f>y?e=g?f-y+e:d-w:!M&&y+r>f+b&&(e=g?f+b-r-y+e:w-r),g||(S>h&&I+h>v+S?i+=v+S-h-I:(h>=S||v>I)&&(i+=v-I)),g&&(I+h>v+S+.49||v>I)||!g&&h>S+.49){var x=this;t.dataSM("scroll-arrows")||t.dataSM("scroll-arrows",$([$('')[0],$('')[0]]).on({mouseenter:function(){t.dataSM("scroll").up=$(this).hasClass("scroll-up"),x.menuScroll(t)},mouseleave:function(e){x.menuScrollStop(t),x.menuScrollOut(t,e)},"mousewheel DOMMouseScroll":function(t){t.preventDefault()}}).insertAfter(t));var A=".smartmenus_scroll";if(t.dataSM("scroll",{y:this.cssTransforms3d?0:i-m,step:1,itemH:m,subH:h,arrowDownH:this.getHeight(t.dataSM("scroll-arrows").eq(1))}).on(getEventsNS({mouseover:function(e){x.menuScrollOver(t,e)},mouseout:function(e){x.menuScrollOut(t,e)},"mousewheel DOMMouseScroll":function(e){x.menuScrollMousewheel(t,e)}},A)).dataSM("scroll-arrows").css({top:"auto",left:"0",marginLeft:e+(parseInt(t.css("border-left-width"))||0),width:r-(parseInt(t.css("border-left-width"))||0)-(parseInt(t.css("border-right-width"))||0),zIndex:t.css("z-index")}).eq(g&&this.opts.bottomToTopSubMenus?0:1).show(),this.isFixed()){var C={};C[touchEvents?"touchstart touchmove touchend":"pointerdown pointermove pointerup MSPointerDown MSPointerMove MSPointerUp"]=function(e){x.menuScrollTouch(t,e)},t.css({"touch-action":"none","-ms-touch-action":"none"}).on(getEventsNS(C,A))}}}t.css({top:"auto",left:"0",marginLeft:e,marginTop:i-m})},menuScroll:function(t,e,i){var s,o=t.dataSM("scroll"),a=t.dataSM("scroll-arrows"),n=o.up?o.upEnd:o.downEnd;if(!e&&o.momentum){if(o.momentum*=.92,s=o.momentum,.5>s)return this.menuScrollStop(t),void 0}else s=i||(e||!this.opts.scrollAccelerate?this.opts.scrollStep:Math.floor(o.step));var r=t.dataSM("level");if(this.activatedItems[r-1]&&this.activatedItems[r-1].dataSM("sub")&&this.activatedItems[r-1].dataSM("sub").is(":visible")&&this.menuHideSubMenus(r-1),o.y=o.up&&o.y>=n||!o.up&&n>=o.y?o.y:Math.abs(n-o.y)>s?o.y+(o.up?s:-s):n,t.css(this.cssTransforms3d?{"-webkit-transform":"translate3d(0, "+o.y+"px, 0)",transform:"translate3d(0, "+o.y+"px, 0)"}:{marginTop:o.y}),mouse&&(o.up&&o.y>o.downEnd||!o.up&&o.y0;t.dataSM("scroll-arrows").eq(i?0:1).is(":visible")&&(t.dataSM("scroll").up=i,this.menuScroll(t,!0))}e.preventDefault()},menuScrollOut:function(t,e){mouse&&(/^scroll-(up|down)/.test((e.relatedTarget||"").className)||(t[0]==e.relatedTarget||$.contains(t[0],e.relatedTarget))&&this.getClosestMenu(e.relatedTarget)==t[0]||t.dataSM("scroll-arrows").css("visibility","hidden"))},menuScrollOver:function(t,e){if(mouse&&!/^scroll-(up|down)/.test(e.target.className)&&this.getClosestMenu(e.target)==t[0]){this.menuScrollRefreshData(t);var i=t.dataSM("scroll"),s=$(window).scrollTop()-t.dataSM("parent-a").offset().top-i.itemH;t.dataSM("scroll-arrows").eq(0).css("margin-top",s).end().eq(1).css("margin-top",s+this.getViewportHeight()-i.arrowDownH).end().css("visibility","visible")}},menuScrollRefreshData:function(t){var e=t.dataSM("scroll"),i=$(window).scrollTop()-t.dataSM("parent-a").offset().top-e.itemH;this.cssTransforms3d&&(i=-(parseFloat(t.css("margin-top"))-i)),$.extend(e,{upEnd:i,downEnd:i+this.getViewportHeight()-e.subH})},menuScrollStop:function(t){return this.scrollTimeout?(cancelAnimationFrame(this.scrollTimeout),this.scrollTimeout=0,t.dataSM("scroll").step=1,!0):void 0},menuScrollTouch:function(t,e){if(e=e.originalEvent,isTouchEvent(e)){var i=this.getTouchPoint(e);if(this.getClosestMenu(i.target)==t[0]){var s=t.dataSM("scroll");if(/(start|down)$/i.test(e.type))this.menuScrollStop(t)?(e.preventDefault(),this.$touchScrollingSub=t):this.$touchScrollingSub=null,this.menuScrollRefreshData(t),$.extend(s,{touchStartY:i.pageY,touchStartTime:e.timeStamp});else if(/move$/i.test(e.type)){var o=void 0!==s.touchY?s.touchY:s.touchStartY;if(void 0!==o&&o!=i.pageY){this.$touchScrollingSub=t;var a=i.pageY>o;void 0!==s.up&&s.up!=a&&$.extend(s,{touchStartY:i.pageY,touchStartTime:e.timeStamp}),$.extend(s,{up:a,touchY:i.pageY}),this.menuScroll(t,!0,Math.abs(i.pageY-o))}e.preventDefault()}else void 0!==s.touchY&&((s.momentum=15*Math.pow(Math.abs(i.pageY-s.touchStartY)/(e.timeStamp-s.touchStartTime),2))&&(this.menuScrollStop(t),this.menuScroll(t),e.preventDefault()),delete s.touchY)}}},menuShow:function(t){if((t.dataSM("beforefirstshowfired")||(t.dataSM("beforefirstshowfired",!0),this.$root.triggerHandler("beforefirstshow.smapi",t[0])!==!1))&&this.$root.triggerHandler("beforeshow.smapi",t[0])!==!1&&(t.dataSM("shown-before",!0),canAnimate&&t.stop(!0,!0),!t.is(":visible"))){var e=t.dataSM("parent-a"),i=this.isCollapsible();if((this.opts.keepHighlighted||i)&&e.addClass("highlighted"),i)t.removeClass("sm-nowrap").css({zIndex:"",width:"auto",minWidth:"",maxWidth:"",top:"",left:"",marginLeft:"",marginTop:""});else{if(t.css("z-index",this.zIndexInc=(this.zIndexInc||this.getStartZIndex())+1),(this.opts.subMenusMinWidth||this.opts.subMenusMaxWidth)&&(t.css({width:"auto",minWidth:"",maxWidth:""}).addClass("sm-nowrap"),this.opts.subMenusMinWidth&&t.css("min-width",this.opts.subMenusMinWidth),this.opts.subMenusMaxWidth)){var s=this.getWidth(t);t.css("max-width",this.opts.subMenusMaxWidth),s>this.getWidth(t)&&t.removeClass("sm-nowrap").css("width",this.opts.subMenusMaxWidth)}this.menuPosition(t)}var o=function(){t.css("overflow","")};i?canAnimate&&this.opts.collapsibleShowFunction?this.opts.collapsibleShowFunction.call(this,t,o):t.show(this.opts.collapsibleShowDuration,o):canAnimate&&this.opts.showFunction?this.opts.showFunction.call(this,t,o):t.show(this.opts.showDuration,o),e.attr("aria-expanded","true"),t.attr({"aria-expanded":"true","aria-hidden":"false"}),this.visibleSubMenus.push(t),this.$root.triggerHandler("show.smapi",t[0])}},popupHide:function(t){this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0);var e=this;this.hideTimeout=setTimeout(function(){e.menuHideAll()},t?1:this.opts.hideTimeout)},popupShow:function(t,e){if(!this.opts.isPopup)return alert('SmartMenus jQuery Error:\n\nIf you want to show this menu via the "popupShow" method, set the isPopup:true option.'),void 0;if(this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0),this.$root.dataSM("shown-before",!0),canAnimate&&this.$root.stop(!0,!0),!this.$root.is(":visible")){this.$root.css({left:t,top:e});var i=this,s=function(){i.$root.css("overflow","")};canAnimate&&this.opts.showFunction?this.opts.showFunction.call(this,this.$root,s):this.$root.show(this.opts.showDuration,s),this.visibleSubMenus[0]=this.$root}},refresh:function(){this.destroy(!0),this.init(!0)},rootKeyDown:function(t){if(this.handleEvents())switch(t.keyCode){case 27:var e=this.activatedItems[0];if(e){this.menuHideAll(),e[0].focus();var i=e.dataSM("sub");i&&this.menuHide(i)}break;case 32:var s=$(t.target);if(s.is("a")&&this.handleItemEvents(s)){var i=s.dataSM("sub");i&&!i.is(":visible")&&(this.itemClick({currentTarget:t.target}),t.preventDefault())}}},rootOut:function(t){if(this.handleEvents()&&!this.isTouchMode()&&t.target!=this.$root[0]&&(this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0),!this.opts.showOnClick||!this.opts.hideOnClick)){var e=this;this.hideTimeout=setTimeout(function(){e.menuHideAll()},this.opts.hideTimeout)}},rootOver:function(t){this.handleEvents()&&!this.isTouchMode()&&t.target!=this.$root[0]&&this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0)},winResize:function(t){if(this.handleEvents()){if(!("onorientationchange"in window)||"orientationchange"==t.type){var e=this.isCollapsible();this.wasCollapsible&&e||(this.activatedItems.length&&this.activatedItems[this.activatedItems.length-1][0].blur(),this.menuHideAll()),this.wasCollapsible=e}}else if(this.$disableOverlay){var i=this.$root.offset();this.$disableOverlay.css({top:i.top,left:i.left,width:this.$root.outerWidth(),height:this.$root.outerHeight()})}}}}),$.fn.dataSM=function(t,e){return e?this.data(t+"_smartmenus",e):this.data(t+"_smartmenus")},$.fn.removeDataSM=function(t){return this.removeData(t+"_smartmenus")},$.fn.smartmenus=function(options){if("string"==typeof options){var args=arguments,method=options;return Array.prototype.shift.call(args),this.each(function(){var t=$(this).data("smartmenus");t&&t[method]&&t[method].apply(t,args)})}return this.each(function(){var dataOpts=$(this).data("sm-options")||null;if(dataOpts)try{dataOpts=eval("("+dataOpts+")")}catch(e){dataOpts=null,alert('ERROR\n\nSmartMenus jQuery init:\nInvalid "data-sm-options" attribute value syntax.')}new $.SmartMenus(this,$.extend({},$.fn.smartmenus.defaults,options,dataOpts))})},$.fn.smartmenus.defaults={isPopup:!1,mainMenuSubOffsetX:0,mainMenuSubOffsetY:0,subMenusSubOffsetX:0,subMenusSubOffsetY:0,subMenusMinWidth:"10em",subMenusMaxWidth:"20em",subIndicators:!0,subIndicatorsPos:"append",subIndicatorsText:"",scrollStep:30,scrollAccelerate:!0,showTimeout:250,hideTimeout:500,showDuration:0,showFunction:null,hideDuration:0,hideFunction:function(t,e){t.fadeOut(200,e)},collapsibleShowDuration:0,collapsibleShowFunction:function(t,e){t.slideDown(200,e)},collapsibleHideDuration:0,collapsibleHideFunction:function(t,e){t.slideUp(200,e)},showOnClick:!1,hideOnClick:!0,noMouseOver:!1,keepInViewport:!0,keepHighlighted:!0,markCurrentItem:!1,markCurrentTree:!0,rightToLeftSubMenus:!1,bottomToTopSubMenus:!1,collapsibleBehavior:"default"},$}); \ No newline at end of file diff --git a/docs/nav_fd.png b/docs/nav_fd.png new file mode 100644 index 0000000000000000000000000000000000000000..032fbdd4c54f54fa9a2e6423b94ef4b2ebdfaceb Binary files /dev/null and b/docs/nav_fd.png differ diff --git a/docs/nav_hd.png b/docs/nav_hd.png new file mode 100644 index 0000000000000000000000000000000000000000..de80f18ad6488b9990303f267a76fdc83f0ffd80 Binary files /dev/null and b/docs/nav_hd.png differ diff --git a/docs/navtree.css b/docs/navtree.css new file mode 100644 index 0000000000000000000000000000000000000000..c8a7766a72248ca187f1a2b936fae239dd99f606 --- /dev/null +++ b/docs/navtree.css @@ -0,0 +1,150 @@ +#nav-tree .children_ul { + margin:0; + padding:4px; +} + +#nav-tree ul { + list-style:none outside none; + margin:0px; + padding:0px; +} + +#nav-tree li { + white-space:nowrap; + margin:0px; + padding:0px; +} + +#nav-tree .plus { + margin:0px; +} + +#nav-tree .selected { + background-image: url('tab_a.png'); + background-repeat:repeat-x; + color: var(--nav-text-active-color); + text-shadow: var(--nav-text-active-shadow); +} + +#nav-tree .selected .arrow { + color: var(--nav-arrow-selected-color); + text-shadow: none; +} + +#nav-tree img { + margin:0px; + padding:0px; + border:0px; + vertical-align: middle; +} + +#nav-tree a { + text-decoration:none; + padding:0px; + margin:0px; + outline:none; +} + +#nav-tree .label { + margin:0px; + padding:0px; + font: 12px var(--font-family-nav); +} + +#nav-tree .label a { + padding:2px; +} + +#nav-tree .selected a { + text-decoration:none; + color:var(--nav-text-active-color); +} + +#nav-tree .children_ul { + margin:0px; + padding:0px; +} + +#nav-tree .item { + margin:0px; + padding:0px; +} + +#nav-tree { + padding: 0px 0px; + font-size:14px; + overflow:auto; +} + +#doc-content { + overflow:auto; + display:block; + padding:0px; + margin:0px; + -webkit-overflow-scrolling : touch; /* iOS 5+ */ +} + +#side-nav { + padding:0 6px 0 0; + margin: 0px; + display:block; + position: absolute; + left: 0px; + width: $width; + overflow : hidden; +} + +.ui-resizable .ui-resizable-handle { + display:block; +} + +.ui-resizable-e { + background-image:var(--nav-splitbar-image); + background-size:100%; + background-repeat:repeat-y; + background-attachment: scroll; + cursor:ew-resize; + height:100%; + right:0; + top:0; + width:6px; +} + +.ui-resizable-handle { + display:none; + font-size:0.1px; + position:absolute; + z-index:1; +} + +#nav-tree-contents { + margin: 6px 0px 0px 0px; +} + +#nav-tree { + background-repeat:repeat-x; + background-color: var(--nav-background-color); + -webkit-overflow-scrolling : touch; /* iOS 5+ */ +} + +#nav-sync { + position:absolute; + top:5px; + right:24px; + z-index:0; +} + +#nav-sync img { + opacity:0.3; +} + +#nav-sync img:hover { + opacity:0.9; +} + +@media print +{ + #nav-tree { display: none; } + div.ui-resizable-handle { display: none; position: relative; } +} + diff --git a/docs/navtree.js b/docs/navtree.js new file mode 100644 index 0000000000000000000000000000000000000000..f23e984956b455e608495f28c314342182f6e499 --- /dev/null +++ b/docs/navtree.js @@ -0,0 +1,556 @@ +/* + @licstart The following is the entire license notice for the JavaScript code in this file. + + The MIT License (MIT) + + Copyright (C) 1997-2020 by Dimitri van Heesch + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + @licend The above is the entire license notice for the JavaScript code in this file + */ +var navTreeSubIndices = new Array(); +var arrowDown = '▼'; +var arrowRight = '►'; + +function getData(varName) +{ + var i = varName.lastIndexOf('/'); + var n = i>=0 ? varName.substring(i+1) : varName; + return eval(n.replace(/\-/g,'_')); +} + +function stripPath(uri) +{ + return uri.substring(uri.lastIndexOf('/')+1); +} + +function stripPath2(uri) +{ + var i = uri.lastIndexOf('/'); + var s = uri.substring(i+1); + var m = uri.substring(0,i+1).match(/\/d\w\/d\w\w\/$/); + return m ? uri.substring(i-6) : s; +} + +function hashValue() +{ + return $(location).attr('hash').substring(1).replace(/[^\w\-]/g,''); +} + +function hashUrl() +{ + return '#'+hashValue(); +} + +function pathName() +{ + return $(location).attr('pathname').replace(/[^-A-Za-z0-9+&@#/%?=~_|!:,.;\(\)]/g, ''); +} + +function localStorageSupported() +{ + try { + return 'localStorage' in window && window['localStorage'] !== null && window.localStorage.getItem; + } + catch(e) { + return false; + } +} + +function storeLink(link) +{ + if (!$("#nav-sync").hasClass('sync') && localStorageSupported()) { + window.localStorage.setItem('navpath',link); + } +} + +function deleteLink() +{ + if (localStorageSupported()) { + window.localStorage.setItem('navpath',''); + } +} + +function cachedLink() +{ + if (localStorageSupported()) { + return window.localStorage.getItem('navpath'); + } else { + return ''; + } +} + +function getScript(scriptName,func,show) +{ + var head = document.getElementsByTagName("head")[0]; + var script = document.createElement('script'); + script.id = scriptName; + script.type = 'text/javascript'; + script.onload = func; + script.src = scriptName+'.js'; + head.appendChild(script); +} + +function createIndent(o,domNode,node,level) +{ + var level=-1; + var n = node; + while (n.parentNode) { level++; n=n.parentNode; } + if (node.childrenData) { + var imgNode = document.createElement("span"); + imgNode.className = 'arrow'; + imgNode.style.paddingLeft=(16*level).toString()+'px'; + imgNode.innerHTML=arrowRight; + node.plus_img = imgNode; + node.expandToggle = document.createElement("a"); + node.expandToggle.href = "javascript:void(0)"; + node.expandToggle.onclick = function() { + if (node.expanded) { + $(node.getChildrenUL()).slideUp("fast"); + node.plus_img.innerHTML=arrowRight; + node.expanded = false; + } else { + expandNode(o, node, false, false); + } + } + node.expandToggle.appendChild(imgNode); + domNode.appendChild(node.expandToggle); + } else { + var span = document.createElement("span"); + span.className = 'arrow'; + span.style.width = 16*(level+1)+'px'; + span.innerHTML = ' '; + domNode.appendChild(span); + } +} + +var animationInProgress = false; + +function gotoAnchor(anchor,aname,updateLocation) +{ + var pos, docContent = $('#doc-content'); + var ancParent = $(anchor.parent()); + if (ancParent.hasClass('memItemLeft') || + ancParent.hasClass('memtitle') || + ancParent.hasClass('fieldname') || + ancParent.hasClass('fieldtype') || + ancParent.is(':header')) + { + pos = ancParent.position().top; + } else if (anchor.position()) { + pos = anchor.position().top; + } + if (pos) { + var dist = Math.abs(Math.min( + pos-docContent.offset().top, + docContent[0].scrollHeight- + docContent.height()-docContent.scrollTop())); + animationInProgress=true; + docContent.animate({ + scrollTop: pos + docContent.scrollTop() - docContent.offset().top + },Math.max(50,Math.min(500,dist)),function(){ + if (updateLocation) window.location.href=aname; + animationInProgress=false; + }); + } +} + +function newNode(o, po, text, link, childrenData, lastNode) +{ + var node = new Object(); + node.children = Array(); + node.childrenData = childrenData; + node.depth = po.depth + 1; + node.relpath = po.relpath; + node.isLast = lastNode; + + node.li = document.createElement("li"); + po.getChildrenUL().appendChild(node.li); + node.parentNode = po; + + node.itemDiv = document.createElement("div"); + node.itemDiv.className = "item"; + + node.labelSpan = document.createElement("span"); + node.labelSpan.className = "label"; + + createIndent(o,node.itemDiv,node,0); + node.itemDiv.appendChild(node.labelSpan); + node.li.appendChild(node.itemDiv); + + var a = document.createElement("a"); + node.labelSpan.appendChild(a); + node.label = document.createTextNode(text); + node.expanded = false; + a.appendChild(node.label); + if (link) { + var url; + if (link.substring(0,1)=='^') { + url = link.substring(1); + link = url; + } else { + url = node.relpath+link; + } + a.className = stripPath(link.replace('#',':')); + if (link.indexOf('#')!=-1) { + var aname = '#'+link.split('#')[1]; + var srcPage = stripPath(pathName()); + var targetPage = stripPath(link.split('#')[0]); + a.href = srcPage!=targetPage ? url : "javascript:void(0)"; + a.onclick = function(){ + storeLink(link); + if (!$(a).parent().parent().hasClass('selected')) + { + $('.item').removeClass('selected'); + $('.item').removeAttr('id'); + $(a).parent().parent().addClass('selected'); + $(a).parent().parent().attr('id','selected'); + } + var anchor = $(aname); + gotoAnchor(anchor,aname,true); + }; + } else { + a.href = url; + a.onclick = function() { storeLink(link); } + } + } else { + if (childrenData != null) + { + a.className = "nolink"; + a.href = "javascript:void(0)"; + a.onclick = node.expandToggle.onclick; + } + } + + node.childrenUL = null; + node.getChildrenUL = function() { + if (!node.childrenUL) { + node.childrenUL = document.createElement("ul"); + node.childrenUL.className = "children_ul"; + node.childrenUL.style.display = "none"; + node.li.appendChild(node.childrenUL); + } + return node.childrenUL; + }; + + return node; +} + +function showRoot() +{ + var headerHeight = $("#top").height(); + var footerHeight = $("#nav-path").height(); + var windowHeight = $(window).height() - headerHeight - footerHeight; + (function (){ // retry until we can scroll to the selected item + try { + var navtree=$('#nav-tree'); + navtree.scrollTo('#selected',100,{offset:-windowHeight/2}); + } catch (err) { + setTimeout(arguments.callee, 0); + } + })(); +} + +function expandNode(o, node, imm, showRoot) +{ + if (node.childrenData && !node.expanded) { + if (typeof(node.childrenData)==='string') { + var varName = node.childrenData; + getScript(node.relpath+varName,function(){ + node.childrenData = getData(varName); + expandNode(o, node, imm, showRoot); + }, showRoot); + } else { + if (!node.childrenVisited) { + getNode(o, node); + } + $(node.getChildrenUL()).slideDown("fast"); + node.plus_img.innerHTML = arrowDown; + node.expanded = true; + } + } +} + +function glowEffect(n,duration) +{ + n.addClass('glow').delay(duration).queue(function(next){ + $(this).removeClass('glow');next(); + }); +} + +function highlightAnchor() +{ + var aname = hashUrl(); + var anchor = $(aname); + if (anchor.parent().attr('class')=='memItemLeft'){ + var rows = $('.memberdecls tr[class$="'+hashValue()+'"]'); + glowEffect(rows.children(),300); // member without details + } else if (anchor.parent().attr('class')=='fieldname'){ + glowEffect(anchor.parent().parent(),1000); // enum value + } else if (anchor.parent().attr('class')=='fieldtype'){ + glowEffect(anchor.parent().parent(),1000); // struct field + } else if (anchor.parent().is(":header")) { + glowEffect(anchor.parent(),1000); // section header + } else { + glowEffect(anchor.next(),1000); // normal member + } +} + +function selectAndHighlight(hash,n) +{ + var a; + if (hash) { + var link=stripPath(pathName())+':'+hash.substring(1); + a=$('.item a[class$="'+link+'"]'); + } + if (a && a.length) { + a.parent().parent().addClass('selected'); + a.parent().parent().attr('id','selected'); + highlightAnchor(); + } else if (n) { + $(n.itemDiv).addClass('selected'); + $(n.itemDiv).attr('id','selected'); + } + var topOffset=5; + if (typeof page_layout!=='undefined' && page_layout==1) { + topOffset+=$('#top').outerHeight(); + } + if ($('#nav-tree-contents .item:first').hasClass('selected')) { + topOffset+=25; + } + $('#nav-sync').css('top',topOffset+'px'); + showRoot(); +} + +function showNode(o, node, index, hash) +{ + if (node && node.childrenData) { + if (typeof(node.childrenData)==='string') { + var varName = node.childrenData; + getScript(node.relpath+varName,function(){ + node.childrenData = getData(varName); + showNode(o,node,index,hash); + },true); + } else { + if (!node.childrenVisited) { + getNode(o, node); + } + $(node.getChildrenUL()).css({'display':'block'}); + node.plus_img.innerHTML = arrowDown; + node.expanded = true; + var n = node.children[o.breadcrumbs[index]]; + if (index+11) hash = '#'+parts[1].replace(/[^\w\-]/g,''); + else hash=''; + } + if (hash.match(/^#l\d+$/)) { + var anchor=$('a[name='+hash.substring(1)+']'); + glowEffect(anchor.parent(),1000); // line number + hash=''; // strip line number anchors + } + var url=root+hash; + var i=-1; + while (NAVTREEINDEX[i+1]<=url) i++; + if (i==-1) { i=0; root=NAVTREE[0][1]; } // fallback: show index + if (navTreeSubIndices[i]) { + gotoNode(o,i,root,hash,relpath) + } else { + getScript(relpath+'navtreeindex'+i,function(){ + navTreeSubIndices[i] = eval('NAVTREEINDEX'+i); + if (navTreeSubIndices[i]) { + gotoNode(o,i,root,hash,relpath); + } + },true); + } +} + +function showSyncOff(n,relpath) +{ + n.html(''); +} + +function showSyncOn(n,relpath) +{ + n.html(''); +} + +function toggleSyncButton(relpath) +{ + var navSync = $('#nav-sync'); + if (navSync.hasClass('sync')) { + navSync.removeClass('sync'); + showSyncOff(navSync,relpath); + storeLink(stripPath2(pathName())+hashUrl()); + } else { + navSync.addClass('sync'); + showSyncOn(navSync,relpath); + deleteLink(); + } +} + +var loadTriggered = false; +var readyTriggered = false; +var loadObject,loadToRoot,loadUrl,loadRelPath; + +$(window).on('load',function(){ + if (readyTriggered) { // ready first + navTo(loadObject,loadToRoot,loadUrl,loadRelPath); + showRoot(); + } + loadTriggered=true; +}); + +function initNavTree(toroot,relpath) +{ + var o = new Object(); + o.toroot = toroot; + o.node = new Object(); + o.node.li = document.getElementById("nav-tree-contents"); + o.node.childrenData = NAVTREE; + o.node.children = new Array(); + o.node.childrenUL = document.createElement("ul"); + o.node.getChildrenUL = function() { return o.node.childrenUL; }; + o.node.li.appendChild(o.node.childrenUL); + o.node.depth = 0; + o.node.relpath = relpath; + o.node.expanded = false; + o.node.isLast = true; + o.node.plus_img = document.createElement("span"); + o.node.plus_img.className = 'arrow'; + o.node.plus_img.innerHTML = arrowRight; + + if (localStorageSupported()) { + var navSync = $('#nav-sync'); + if (cachedLink()) { + showSyncOff(navSync,relpath); + navSync.removeClass('sync'); + } else { + showSyncOn(navSync,relpath); + } + navSync.click(function(){ toggleSyncButton(relpath); }); + } + + if (loadTriggered) { // load before ready + navTo(o,toroot,hashUrl(),relpath); + showRoot(); + } else { // ready before load + loadObject = o; + loadToRoot = toroot; + loadUrl = hashUrl(); + loadRelPath = relpath; + readyTriggered=true; + } + + $(window).bind('hashchange', function(){ + if (window.location.hash && window.location.hash.length>1){ + var a; + if ($(location).attr('hash')){ + var clslink=stripPath(pathName())+':'+hashValue(); + a=$('.item a[class$="'+clslink.replace(/0) { + newWidth=0; + } + else { + var width = readSetting('width'); + newWidth = (width>250 && width<$(window).width()) ? width : 250; + } + restoreWidth(newWidth); + var sidenavWidth = $(sidenav).outerWidth(); + writeSetting('width',sidenavWidth-barWidth); + } + + header = $("#top"); + sidenav = $("#side-nav"); + content = $("#doc-content"); + navtree = $("#nav-tree"); + footer = $("#nav-path"); + $(".side-nav-resizable").resizable({resize: function(e, ui) { resizeWidth(); } }); + $(sidenav).resizable({ minWidth: 0 }); + $(window).resize(function() { resizeHeight(); }); + var device = navigator.userAgent.toLowerCase(); + var touch_device = device.match(/(iphone|ipod|ipad|android)/); + if (touch_device) { /* wider split bar for touch only devices */ + $(sidenav).css({ paddingRight:'20px' }); + $('.ui-resizable-e').css({ width:'20px' }); + $('#nav-sync').css({ right:'34px' }); + barWidth=20; + } + var width = readSetting('width'); + if (width) { restoreWidth(width); } else { resizeWidth(); } + resizeHeight(); + var url = location.href; + var i=url.indexOf("#"); + if (i>=0) window.location.hash=url.substr(i); + var _preventDefault = function(evt) { evt.preventDefault(); }; + $("#splitbar").bind("dragstart", _preventDefault).bind("selectstart", _preventDefault); + if (once) { + $(".ui-resizable-handle").dblclick(collapseExpand); + once=0 + } + $(window).on('load',resizeHeight); +} +/* @license-end */ diff --git a/docs/search/all_0.js b/docs/search/all_0.js new file mode 100644 index 0000000000000000000000000000000000000000..07f86d2788e5d60c1780084f99dfcf37f1d8e63f --- /dev/null +++ b/docs/search/all_0.js @@ -0,0 +1,16 @@ +var searchData= +[ + ['_5f_5fforceinline_0',['__forceinline',['../intel-ipsec-mb_8h.html#af93b819ac40799ac392e16f6a90729fd',1,'intel-ipsec-mb.h']]], + ['_5fhashed_5fauth_5fkey_5fxor_5fipad_1',['_hashed_auth_key_xor_ipad',['../structIMB__JOB.html#a2e74bbeeed2c70f97e6b30a0a12aec0e',1,'IMB_JOB']]], + ['_5fhashed_5fauth_5fkey_5fxor_5fopad_2',['_hashed_auth_key_xor_opad',['../structIMB__JOB.html#a684b751d0a275a28d345a3d1051204ef',1,'IMB_JOB']]], + ['_5finit_5ftag_3',['_init_tag',['../structIMB__JOB.html#a1e2fc881ce9c84173168df1dbda851f8',1,'IMB_JOB']]], + ['_5fiv_4',['_iv',['../structIMB__JOB.html#ae571147592c332cea0d90a36e133e6a5',1,'IMB_JOB::_iv'],['../structIMB__JOB.html#a4415e6851e770734c5ec1b46a828ce55',1,'IMB_JOB::_iv']]], + ['_5fiv23_5',['_iv23',['../structIMB__JOB.html#a2bd093b1609dad73bbef2a728995d12f',1,'IMB_JOB']]], + ['_5fk1_5fexpanded_6',['_k1_expanded',['../structIMB__JOB.html#a76d55a988c71e40c577c6f8f5d0496de',1,'IMB_JOB']]], + ['_5fk2_7',['_k2',['../structIMB__JOB.html#adc47d4af15c88dcb7abf01bb07e79e63',1,'IMB_JOB']]], + ['_5fk3_8',['_k3',['../structIMB__JOB.html#afa62af4d5f1ecbb80f5216ec298fcfe8',1,'IMB_JOB']]], + ['_5fkey_9',['_key',['../structIMB__JOB.html#a7ba1fdbcc6ed2e3840990d2e34dac8e8',1,'IMB_JOB::_key'],['../structIMB__JOB.html#a7a62dc4b2ef35231a93ce035561c4153',1,'IMB_JOB::_key'],['../structIMB__JOB.html#a3b3f616759b854553a351e6b44e08471',1,'IMB_JOB::_key']]], + ['_5fkey_5fexpanded_10',['_key_expanded',['../structIMB__JOB.html#a95cc77e0c079f43dec1a46a2b75422a0',1,'IMB_JOB']]], + ['_5fskey1_11',['_skey1',['../structIMB__JOB.html#ab64e8f6bf8dfaf56305b9023103ea5e2',1,'IMB_JOB']]], + ['_5fskey2_12',['_skey2',['../structIMB__JOB.html#a8e8143e461eab3b2b04333c5f1c17a82',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_1.js b/docs/search/all_1.js new file mode 100644 index 0000000000000000000000000000000000000000..afb95cfe3c0fc816e20af46bf7ac3b00f53c7f96 --- /dev/null +++ b/docs/search/all_1.js @@ -0,0 +1,151 @@ +var searchData= +[ + ['aad_0',['aad',['../structIMB__JOB.html#a4c943fcac7f5c4f5d6d030547840071e',1,'IMB_JOB']]], + ['aad_5fhash_1',['aad_hash',['../structgcm__context__data.html#a98c22f35eb0b4d368e2f454b347ed3b9',1,'gcm_context_data']]], + ['aad_5flen_2',['aad_len',['../structchacha20__poly1305__context__data.html#a7b5edcebc9a6bec3776daf6d13bf950c',1,'chacha20_poly1305_context_data']]], + ['aad_5flen_5fin_5fbytes_3',['aad_len_in_bytes',['../structIMB__JOB.html#aa607a780bc2c8509270625c7520a11f1',1,'IMB_JOB']]], + ['aad_5flength_4',['aad_length',['../structgcm__context__data.html#ab1dadabcbd2002738ec7db7da332e04f',1,'gcm_context_data']]], + ['aes128_5fcbcs_5fooo_5',['aes128_cbcs_ooo',['../structIMB__MGR.html#a9c72f3341a9a4837595338d06c3eda5c',1,'IMB_MGR']]], + ['aes128_5fcfb_5fone_6',['aes128_cfb_one',['../structIMB__MGR.html#abd468de03563a40da0b21b0885f44cbe',1,'IMB_MGR']]], + ['aes128_5fooo_7',['aes128_ooo',['../structIMB__MGR.html#a97fd05a4057fc898f647955a7ba8501b',1,'IMB_MGR']]], + ['aes192_5fooo_8',['aes192_ooo',['../structIMB__MGR.html#afe48187946dff59f6c69103501691921',1,'IMB_MGR']]], + ['aes256_5fccm_5fooo_9',['aes256_ccm_ooo',['../structIMB__MGR.html#a04c28bfe763544b68dac98e23aa0a8d9',1,'IMB_MGR']]], + ['aes256_5fcfb_5fone_10',['aes256_cfb_one',['../structIMB__MGR.html#ab4e116827875a9dd2573f425081c7eaf',1,'IMB_MGR']]], + ['aes256_5fcmac_5fooo_11',['aes256_cmac_ooo',['../structIMB__MGR.html#acbd18bab8176262daab2bd808ca30616',1,'IMB_MGR']]], + ['aes256_5fooo_12',['aes256_ooo',['../structIMB__MGR.html#a6abbf129001cb44003a10105115e52b7',1,'IMB_MGR']]], + ['aes_5fccm_5fooo_13',['aes_ccm_ooo',['../structIMB__MGR.html#a3eeeead99056e46d8cb1ed316f587aaf',1,'IMB_MGR']]], + ['aes_5fcfb_5ft_14',['aes_cfb_t',['../intel-ipsec-mb_8h.html#aeb837fda81cdf56ffcfa4e68b5877a35',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fooo_15',['aes_cmac_ooo',['../structIMB__MGR.html#a0bd2acac3a6c3f90a998ed53a7d28e57',1,'IMB_MGR']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx_16',['aes_cmac_subkey_gen_avx',['../intel-ipsec-mb_8h.html#a6541825066a8d2b8b3c7b6e2c4b28c83',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx2_17',['aes_cmac_subkey_gen_avx2',['../intel-ipsec-mb_8h.html#a6903e1e0dfb8fd75c2f556596f3efa1b',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx512_18',['aes_cmac_subkey_gen_avx512',['../intel-ipsec-mb_8h.html#a730f18bd7cdcbbb995e24931307867fc',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5fsse_19',['aes_cmac_subkey_gen_sse',['../intel-ipsec-mb_8h.html#a4dafe82f4933670f68e1bf34d55a691d',1,'intel-ipsec-mb.h']]], + ['aes_5fecb_5f128_5fquic_20',['aes_ecb_128_quic',['../structIMB__MGR.html#a436adbd14682855df093afb36c569039',1,'IMB_MGR']]], + ['aes_5fecb_5f256_5fquic_21',['aes_ecb_256_quic',['../structIMB__MGR.html#a5e589dea4f677ad01cfb01d06884a281',1,'IMB_MGR']]], + ['aes_5fecb_5fquic_5ft_22',['aes_ecb_quic_t',['../intel-ipsec-mb_8h.html#ad907e0caa52fcb257275464179620643',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen2_23',['aes_gcm_dec_128_avx_gen2',['../intel-ipsec-mb_8h.html#a8c9a7e51599dbce89a416e0d8422742b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen4_24',['aes_gcm_dec_128_avx_gen4',['../intel-ipsec-mb_8h.html#ae1e8e1b4425af520048a9e1fe5689e4d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen2_25',['aes_gcm_dec_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#ac173e4fb252a820edb25ad790065ad39',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen4_26',['aes_gcm_dec_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a8e2bc4fff9de3c70ac140ff30e1cdce0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5fsse_27',['aes_gcm_dec_128_finalize_sse',['../intel-ipsec-mb_8h.html#a08603861ac18d1dce7b322b0c3fb69c5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fsse_28',['aes_gcm_dec_128_sse',['../intel-ipsec-mb_8h.html#add2f5aafaf55cd0e0b20b2b62fcb025d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen2_29',['aes_gcm_dec_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#a796e1df6d2aa233a6e28a867c1a2c628',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen4_30',['aes_gcm_dec_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#a4464684ff4cf4e9cb978409ad350c4d6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5fsse_31',['aes_gcm_dec_128_update_sse',['../intel-ipsec-mb_8h.html#a2216e461e916668b70d547513b0cf26f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen2_32',['aes_gcm_dec_192_avx_gen2',['../intel-ipsec-mb_8h.html#ad45e83e263506684c786b4e7012055ac',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen4_33',['aes_gcm_dec_192_avx_gen4',['../intel-ipsec-mb_8h.html#a5ccd87fbcfffc238fbcd1fc816eb06ee',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen2_34',['aes_gcm_dec_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#aacc19f04c9ac80d1bf7391661e191798',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen4_35',['aes_gcm_dec_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#afa136a3c06d45e914d15415dee27f001',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5fsse_36',['aes_gcm_dec_192_finalize_sse',['../intel-ipsec-mb_8h.html#a0fd357a1794d78ac2168bc7eff0442ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fsse_37',['aes_gcm_dec_192_sse',['../intel-ipsec-mb_8h.html#acaff83c8ca568201805e0ada5a79f6e1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen2_38',['aes_gcm_dec_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#aa81615f7ddfc8e1a794c5e278d22f7fc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen4_39',['aes_gcm_dec_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a615c26f03ba08c8cde73cb074bf6b363',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5fsse_40',['aes_gcm_dec_192_update_sse',['../intel-ipsec-mb_8h.html#a58455aec5650ecfc6077c69c97031f63',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen2_41',['aes_gcm_dec_256_avx_gen2',['../intel-ipsec-mb_8h.html#a9b79136dc5996baf598e3aeee520871c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen4_42',['aes_gcm_dec_256_avx_gen4',['../intel-ipsec-mb_8h.html#ac16384799933d4f7b11cb56d9c66f5da',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen2_43',['aes_gcm_dec_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a72f76caa84e2803b2b4a26c81623f89e',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen4_44',['aes_gcm_dec_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a791be274839adc3d26d9a32aa319dc3b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5fsse_45',['aes_gcm_dec_256_finalize_sse',['../intel-ipsec-mb_8h.html#adf7aa2e79dce7a9ed79134c086715fd0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fsse_46',['aes_gcm_dec_256_sse',['../intel-ipsec-mb_8h.html#afd8763c66749e9bc5e4e5eb2ac130e43',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen2_47',['aes_gcm_dec_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a0c31c2e2760edc7b252214214159cd7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen4_48',['aes_gcm_dec_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#a141303c38f4eed1245b542b6e9473296',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5fsse_49',['aes_gcm_dec_256_update_sse',['../intel-ipsec-mb_8h.html#ab1a51ae4b3cf3e8fdb618773d7b24b68',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen2_50',['aes_gcm_enc_128_avx_gen2',['../intel-ipsec-mb_8h.html#af33d7afa861dcedc2c284330a5a66164',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen4_51',['aes_gcm_enc_128_avx_gen4',['../intel-ipsec-mb_8h.html#aa17c919e873b6cfd7ac990977eef7601',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen2_52',['aes_gcm_enc_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a40a83ee6dec0440cd99e5ff600e3803d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen4_53',['aes_gcm_enc_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a004c38b922d25d8631252854426ebf44',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5fsse_54',['aes_gcm_enc_128_finalize_sse',['../intel-ipsec-mb_8h.html#af63ae2246c6d89b7d6630623948d3886',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fsse_55',['aes_gcm_enc_128_sse',['../intel-ipsec-mb_8h.html#aad9e6c466a5274b974d20330f91add50',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen2_56',['aes_gcm_enc_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#ae3459d38a2ee2e1ce04b9f9f4bf55ea5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen4_57',['aes_gcm_enc_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#abb5570a161a90391e25b2016bc98381b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5fsse_58',['aes_gcm_enc_128_update_sse',['../intel-ipsec-mb_8h.html#a7b12cabbfebb420022a19124aff78a3f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen2_59',['aes_gcm_enc_192_avx_gen2',['../intel-ipsec-mb_8h.html#a0ff9569b5072896a4119a4c7d3849bfc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen4_60',['aes_gcm_enc_192_avx_gen4',['../intel-ipsec-mb_8h.html#a9b462a3c00471e4338f0bae7d5abbda7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen2_61',['aes_gcm_enc_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a5177757ae4a75ba41d5e7a8a534edc06',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen4_62',['aes_gcm_enc_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a4f76beaee47ffeeeb6f94a132f898339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5fsse_63',['aes_gcm_enc_192_finalize_sse',['../intel-ipsec-mb_8h.html#a5dcb5c37d1a5377967888e9b76208f8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fsse_64',['aes_gcm_enc_192_sse',['../intel-ipsec-mb_8h.html#adec2f9778622b3e226f12bb326e75076',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen2_65',['aes_gcm_enc_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#a548fcbb89ff5fb798bf71ab8e4ac88eb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen4_66',['aes_gcm_enc_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a174447b05661285b2729805f66bcef81',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5fsse_67',['aes_gcm_enc_192_update_sse',['../intel-ipsec-mb_8h.html#a46c3df131f9e1e3ed1a283349c4072f7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen2_68',['aes_gcm_enc_256_avx_gen2',['../intel-ipsec-mb_8h.html#a45dea093699e6e2dd462b6cafccee672',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen4_69',['aes_gcm_enc_256_avx_gen4',['../intel-ipsec-mb_8h.html#a52c0913f10dd41fea5fe328037561ff8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen2_70',['aes_gcm_enc_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a7fa5d4f44bb44600fa7b86697ccc385f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen4_71',['aes_gcm_enc_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a873a75e2175acd3741b5cfc41180fe2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5fsse_72',['aes_gcm_enc_256_finalize_sse',['../intel-ipsec-mb_8h.html#afcdedeabc7516e7506fa5c81d07f6fd8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fsse_73',['aes_gcm_enc_256_sse',['../intel-ipsec-mb_8h.html#a460ad1ebcb7df39e39c9aca214d8109c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen2_74',['aes_gcm_enc_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a351c2bf07dcab7e01570273c9df1e56f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen4_75',['aes_gcm_enc_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#ad9592051cf26814ea5830826b553c547',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5fsse_76',['aes_gcm_enc_256_update_sse',['../intel-ipsec-mb_8h.html#ab831bdb2193a5adc54b51285f23ad5ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5ffinalize_5ft_77',['aes_gcm_enc_dec_finalize_t',['../intel-ipsec-mb_8h.html#a8c85430db7a8b695a2836dcb96653bc5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5fiv_5ft_78',['aes_gcm_enc_dec_iv_t',['../intel-ipsec-mb_8h.html#ac5bfb31d18fdc55451cd325c4af032c7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5ft_79',['aes_gcm_enc_dec_t',['../intel-ipsec-mb_8h.html#a71fb9f25bcab533a98f3d1c5cc2c7f48',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5fupdate_5ft_80',['aes_gcm_enc_dec_update_t',['../intel-ipsec-mb_8h.html#af701adcd98fd95084d76e0cb0f926fef',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen2_81',['aes_gcm_init_128_avx_gen2',['../intel-ipsec-mb_8h.html#a7a9a24d480dcedac1aa40d53e01b8a76',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen4_82',['aes_gcm_init_128_avx_gen4',['../intel-ipsec-mb_8h.html#a8d225d08a5cf144208dacb155e192cb6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5fsse_83',['aes_gcm_init_128_sse',['../intel-ipsec-mb_8h.html#aca7ab49c370f8f7cb82f93db199bfd1c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen2_84',['aes_gcm_init_192_avx_gen2',['../intel-ipsec-mb_8h.html#ac24548c2955399fd84b03216ad5b36fb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen4_85',['aes_gcm_init_192_avx_gen4',['../intel-ipsec-mb_8h.html#a757cd199a2e993c6610bbcdfc1da1ae4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5fsse_86',['aes_gcm_init_192_sse',['../intel-ipsec-mb_8h.html#a5bbf84135701f572ec2ecb0c0898848b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen2_87',['aes_gcm_init_256_avx_gen2',['../intel-ipsec-mb_8h.html#a3dd74260f4f34f7a53253c63ad3a6701',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen4_88',['aes_gcm_init_256_avx_gen4',['../intel-ipsec-mb_8h.html#a99310b73bc9cde6b69b682e72cd0efc1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5fsse_89',['aes_gcm_init_256_sse',['../intel-ipsec-mb_8h.html#a09949bd5112eec14773dce239ca39033',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5ft_90',['aes_gcm_init_t',['../intel-ipsec-mb_8h.html#adeecd4bac067493d8465e8d7b6b7dd6d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5fvar_5fiv_5ft_91',['aes_gcm_init_var_iv_t',['../intel-ipsec-mb_8h.html#a65f909001359ad986640155f8756a11d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen2_92',['aes_gcm_pre_128_avx_gen2',['../intel-ipsec-mb_8h.html#a2f60098fff25e8b71b35e42cc7854339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen4_93',['aes_gcm_pre_128_avx_gen4',['../intel-ipsec-mb_8h.html#a3d7b8bc454b44abbeaa8f435280f8cb1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5fsse_94',['aes_gcm_pre_128_sse',['../intel-ipsec-mb_8h.html#a5d02ac2586befa6cd3fd4921858c33fd',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen2_95',['aes_gcm_pre_192_avx_gen2',['../intel-ipsec-mb_8h.html#a6c9c0b179633aca3ba5c520bee32662f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen4_96',['aes_gcm_pre_192_avx_gen4',['../intel-ipsec-mb_8h.html#ab71c95df9bd059e1a5e108d2513a0e2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5fsse_97',['aes_gcm_pre_192_sse',['../intel-ipsec-mb_8h.html#ac0ebc1e7678bde75c25d5c0ec7ab2f7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen2_98',['aes_gcm_pre_256_avx_gen2',['../intel-ipsec-mb_8h.html#aaaabf4acdd43f98b6c2d31c8013ebbf4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen4_99',['aes_gcm_pre_256_avx_gen4',['../intel-ipsec-mb_8h.html#a082773f106f399ca73afc56583805a8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5fsse_100',['aes_gcm_pre_256_sse',['../intel-ipsec-mb_8h.html#a18fac8ea0f3acabb4d5ffab1152f477f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5ft_101',['aes_gcm_pre_t',['../intel-ipsec-mb_8h.html#a11e10a3ac7a08801dc68949fb0f428af',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen2_102',['aes_gcm_precomp_128_avx_gen2',['../intel-ipsec-mb_8h.html#ade16ed4c31baa50f8a10e3c85156a942',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen4_103',['aes_gcm_precomp_128_avx_gen4',['../intel-ipsec-mb_8h.html#aecf7277999128599c6cf8432b242ba36',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5fsse_104',['aes_gcm_precomp_128_sse',['../intel-ipsec-mb_8h.html#a46253857355ddae8d17bab1403289d0f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen2_105',['aes_gcm_precomp_192_avx_gen2',['../intel-ipsec-mb_8h.html#ab0055e3b7d00cdb578647921207a1de7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen4_106',['aes_gcm_precomp_192_avx_gen4',['../intel-ipsec-mb_8h.html#a6cb54cf9321dfb95af2eeb812f41fa8f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5fsse_107',['aes_gcm_precomp_192_sse',['../intel-ipsec-mb_8h.html#a72d93e3249806eee186d465dc04c09e6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen2_108',['aes_gcm_precomp_256_avx_gen2',['../intel-ipsec-mb_8h.html#a2a883085207ee21caca109ab8250db08',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen4_109',['aes_gcm_precomp_256_avx_gen4',['../intel-ipsec-mb_8h.html#abef922c52013630ad70273cef095c564',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5fsse_110',['aes_gcm_precomp_256_sse',['../intel-ipsec-mb_8h.html#a18001897ea80cdce1f4f608b0922f767',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5ft_111',['aes_gcm_precomp_t',['../intel-ipsec-mb_8h.html#ab50906d68121cb8e6edf2a3493a60c5c',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5ffinalize_5ft_112',['aes_gmac_finalize_t',['../intel-ipsec-mb_8h.html#af3f75897a0ec8b3f22c4e2b01f7f48b7',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5finit_5ft_113',['aes_gmac_init_t',['../intel-ipsec-mb_8h.html#a31987251c968435d720e8656ae4621a0',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5fupdate_5ft_114',['aes_gmac_update_t',['../intel-ipsec-mb_8h.html#aba0ac59064802c6d0a182a5164568c18',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx_115',['aes_keyexp_128_avx',['../intel-ipsec-mb_8h.html#aec7fe23c32ee94de88e5feba8f373a04',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx2_116',['aes_keyexp_128_avx2',['../intel-ipsec-mb_8h.html#a4fc7f88bdb263ae0d45f37af7e4cb930',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx512_117',['aes_keyexp_128_avx512',['../intel-ipsec-mb_8h.html#a9b7faaf3e5235d0a3c4b784f1035896d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx_118',['aes_keyexp_128_enc_avx',['../intel-ipsec-mb_8h.html#ac7d7c889a25730d82ddee27bad7f0a63',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx2_119',['aes_keyexp_128_enc_avx2',['../intel-ipsec-mb_8h.html#a84e76bdfb5a69dba7b515dd0b4fb4312',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx512_120',['aes_keyexp_128_enc_avx512',['../intel-ipsec-mb_8h.html#a68e6a0e99a2091d3630bf3315fe9d95c',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5fsse_121',['aes_keyexp_128_enc_sse',['../intel-ipsec-mb_8h.html#a270621a2f73134d8d021b5dd57ad6817',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fsse_122',['aes_keyexp_128_sse',['../intel-ipsec-mb_8h.html#a32bcda3484b33b7d01124dbfeb9a229b',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx_123',['aes_keyexp_192_avx',['../intel-ipsec-mb_8h.html#a3bc56101b48217094086ae3120b0e086',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx2_124',['aes_keyexp_192_avx2',['../intel-ipsec-mb_8h.html#aba6dd2e633819829740cb7917ba6b1b2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx512_125',['aes_keyexp_192_avx512',['../intel-ipsec-mb_8h.html#a1f69c0a43de83f8fba19107a63c07831',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx_126',['aes_keyexp_192_enc_avx',['../intel-ipsec-mb_8h.html#a5350b81ab574c990d6a5b9dc3b203c51',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx2_127',['aes_keyexp_192_enc_avx2',['../intel-ipsec-mb_8h.html#a460853c072adc3a0f431e861d3e5bba1',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx512_128',['aes_keyexp_192_enc_avx512',['../intel-ipsec-mb_8h.html#a72807e8300208b22b5d532e274385127',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5fsse_129',['aes_keyexp_192_enc_sse',['../intel-ipsec-mb_8h.html#a0a843e722f2af7a645340f6a98cfa4c2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fsse_130',['aes_keyexp_192_sse',['../intel-ipsec-mb_8h.html#a7681d777bb5f991ab59b5581bc854562',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx_131',['aes_keyexp_256_avx',['../intel-ipsec-mb_8h.html#a5b92ee3873b2e2cd02c9312b6040ee91',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx2_132',['aes_keyexp_256_avx2',['../intel-ipsec-mb_8h.html#af1f440855a00edbf95c124a1e1d18c37',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx512_133',['aes_keyexp_256_avx512',['../intel-ipsec-mb_8h.html#a37e0f61656618413fcfd919dfe79ca45',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx_134',['aes_keyexp_256_enc_avx',['../intel-ipsec-mb_8h.html#a22096fdd9b5d32095571f04a838d66a9',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx2_135',['aes_keyexp_256_enc_avx2',['../intel-ipsec-mb_8h.html#a73da5625112ad845a3ec94ee84590f54',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx512_136',['aes_keyexp_256_enc_avx512',['../intel-ipsec-mb_8h.html#a9fb47323b270bffc87ad8e77985e827d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5fsse_137',['aes_keyexp_256_enc_sse',['../intel-ipsec-mb_8h.html#afa369dee8def2ef620e7f33db86a3356',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fsse_138',['aes_keyexp_256_sse',['../intel-ipsec-mb_8h.html#a5105a8d995cd8e4354c7de7ac94886cb',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx_139',['aes_xcbc_expand_key_avx',['../intel-ipsec-mb_8h.html#afe460e4ff6a7b0c7a6edcc87e9976121',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx2_140',['aes_xcbc_expand_key_avx2',['../intel-ipsec-mb_8h.html#ac3f27672ce30d74f5cdc28f1903f68b1',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx512_141',['aes_xcbc_expand_key_avx512',['../intel-ipsec-mb_8h.html#a1308be648182642b8d73ff27d9344564',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5fsse_142',['aes_xcbc_expand_key_sse',['../intel-ipsec-mb_8h.html#aefb28e46620cbf65a2af3073b2401e40',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fooo_143',['aes_xcbc_ooo',['../structIMB__MGR.html#ad554a25793a904ae6cf7ca87250adbd3',1,'IMB_MGR']]], + ['alloc_5fmb_5fmgr_144',['alloc_mb_mgr',['../intel-ipsec-mb_8h.html#ab8dc617feb010b0f4cd74fd9e1eabb5e',1,'intel-ipsec-mb.h']]], + ['auth_5ftag_5foutput_145',['auth_tag_output',['../structIMB__JOB.html#ac37080ddb0baf56c9589e65960430b32',1,'IMB_JOB']]], + ['auth_5ftag_5foutput_5flen_5fin_5fbytes_146',['auth_tag_output_len_in_bytes',['../structIMB__JOB.html#a750df4fa7330e2387138409d0eb50e0f',1,'IMB_JOB']]], + ['avx2_5favx512_147',['avx2_avx512',['../structgcm__key__data.html#a3f231fdc1d521cd15c111667e4aea818',1,'gcm_key_data']]] +]; diff --git a/docs/search/all_10.js b/docs/search/all_10.js new file mode 100644 index 0000000000000000000000000000000000000000..b274ce7e6a871e4502c18d66f949414b83aa8b57 --- /dev/null +++ b/docs/search/all_10.js @@ -0,0 +1,8 @@ +var searchData= +[ + ['partial_5fblock_5fenc_5fkey_0',['partial_block_enc_key',['../structgcm__context__data.html#a2e52ea863c681afe1d3370a987e252f0',1,'gcm_context_data']]], + ['partial_5fblock_5flength_1',['partial_block_length',['../structgcm__context__data.html#a97411247efd14ab01e7e0da6587bfa0c',1,'gcm_context_data']]], + ['poly1305_2',['POLY1305',['../structIMB__JOB.html#a15f9c1a5ab25dca6e3a550ff00ad2419',1,'IMB_JOB']]], + ['poly_5fkey_3',['poly_key',['../structchacha20__poly1305__context__data.html#a9ac4fafe2ca37395264f728c8e99a9b1',1,'chacha20_poly1305_context_data']]], + ['poly_5fscratch_4',['poly_scratch',['../structchacha20__poly1305__context__data.html#a908d80ce2c35792099dbc967818b9abd',1,'chacha20_poly1305_context_data']]] +]; diff --git a/docs/search/all_11.js b/docs/search/all_11.js new file mode 100644 index 0000000000000000000000000000000000000000..a3966989e9fe034a1f6db0ed437bc76b3c400e53 --- /dev/null +++ b/docs/search/all_11.js @@ -0,0 +1,9 @@ +var searchData= +[ + ['queue_5fsize_0',['queue_size',['../structIMB__MGR.html#a5a35c9c2a1c8eb5b4aa239a3b04f4fbf',1,'IMB_MGR']]], + ['queue_5fsize_5favx_1',['queue_size_avx',['../intel-ipsec-mb_8h.html#a800c395d326e718a2b520d0bb91cbb89',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5favx2_2',['queue_size_avx2',['../intel-ipsec-mb_8h.html#a8a540000c08366ad47d95e23ed5b2918',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5favx512_3',['queue_size_avx512',['../intel-ipsec-mb_8h.html#a51c0426a4b57c9e7db2b091692dff472',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5fsse_4',['queue_size_sse',['../intel-ipsec-mb_8h.html#a80f792744558de173ea97d722c58e5c1',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5ft_5',['queue_size_t',['../intel-ipsec-mb_8h.html#a9884ae92b598dd504f7d0ef8be5051d6',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_12.js b/docs/search/all_12.js new file mode 100644 index 0000000000000000000000000000000000000000..ddd86bc57617d87a68242d7befe70d9ca87f3439 --- /dev/null +++ b/docs/search/all_12.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['readme_2emd_0',['README.md',['../README_8md.html',1,'']]], + ['remain_5fct_5fbytes_1',['remain_ct_bytes',['../structchacha20__poly1305__context__data.html#a74e971fe76c8b946e30e8f0c8cce111a',1,'chacha20_poly1305_context_data']]], + ['remain_5fks_5fbytes_2',['remain_ks_bytes',['../structchacha20__poly1305__context__data.html#a84693b96048f6130a749b633c65e2105',1,'chacha20_poly1305_context_data']]], + ['reserved_3',['reserved',['../structIMB__JOB.html#a9d5e67861edf38dfa23160b650c7caa9',1,'IMB_JOB::reserved'],['../structIMB__MGR.html#a136697eb37d5a06eeada0f338abd8b51',1,'IMB_MGR::reserved']]] +]; diff --git a/docs/search/all_13.js b/docs/search/all_13.js new file mode 100644 index 0000000000000000000000000000000000000000..244c601eecdcd150b5ed8b1d1d4955852cc7fbd5 --- /dev/null +++ b/docs/search/all_13.js @@ -0,0 +1,78 @@ +var searchData= +[ + ['session_5fid_0',['session_id',['../structIMB__JOB.html#ab4571657aa5147f768083e8969ea96c9',1,'IMB_JOB']]], + ['set_5fsuite_5fid_1',['set_suite_id',['../structIMB__MGR.html#a95209e74a7acff69d46eef91021a1168',1,'IMB_MGR']]], + ['sgl_5fio_5fsegs_2',['sgl_io_segs',['../structIMB__JOB.html#a2462f9efb16c31ae615df7493dddb343',1,'IMB_JOB']]], + ['sgl_5fstate_3',['sgl_state',['../structIMB__JOB.html#a3d7e363be004999d02db464c721e9751',1,'IMB_JOB']]], + ['sha1_4',['sha1',['../structIMB__MGR.html#a82aaedc800096aaa55062c56131e0b13',1,'IMB_MGR']]], + ['sha1_5fone_5fblock_5',['sha1_one_block',['../structIMB__MGR.html#a80a26be9a12f561a68b70d65e096cde9',1,'IMB_MGR']]], + ['sha224_6',['sha224',['../structIMB__MGR.html#a52e26b8805b28d1298ac48e5762412b5',1,'IMB_MGR']]], + ['sha224_5fone_5fblock_7',['sha224_one_block',['../structIMB__MGR.html#aacf8209094d12ceceed96d4d0fae193c',1,'IMB_MGR']]], + ['sha256_8',['sha256',['../structIMB__MGR.html#a8d978764a953e220b3c799990bae490b',1,'IMB_MGR']]], + ['sha256_5fone_5fblock_9',['sha256_one_block',['../structIMB__MGR.html#ade4a72c39980f0bba790b70529b60158',1,'IMB_MGR']]], + ['sha384_10',['sha384',['../structIMB__MGR.html#a9ffcce0ba9aef3cd68bdd7f6e9e3e2a0',1,'IMB_MGR']]], + ['sha384_5fone_5fblock_11',['sha384_one_block',['../structIMB__MGR.html#a92b7ff145f0016dd46eabf9271ef03f1',1,'IMB_MGR']]], + ['sha512_12',['sha512',['../structIMB__MGR.html#a62ebdd5e2b5a59c9e648511caf56f3f1',1,'IMB_MGR']]], + ['sha512_5fone_5fblock_13',['sha512_one_block',['../structIMB__MGR.html#a88f1060a2adb8098fc085fc2b812addc',1,'IMB_MGR']]], + ['sha_5f1_5fooo_14',['sha_1_ooo',['../structIMB__MGR.html#acdb53e812f3e4a307a497eeab28c4256',1,'IMB_MGR']]], + ['sha_5f224_5fooo_15',['sha_224_ooo',['../structIMB__MGR.html#aa174bf6846067a48f3f47ee5ebd74877',1,'IMB_MGR']]], + ['sha_5f256_5fooo_16',['sha_256_ooo',['../structIMB__MGR.html#a37fc5ccae7af099a8517566a20751265',1,'IMB_MGR']]], + ['sha_5f384_5fooo_17',['sha_384_ooo',['../structIMB__MGR.html#a51a499114f84eed88cb48bd3b461bd1c',1,'IMB_MGR']]], + ['sha_5f512_5fooo_18',['sha_512_ooo',['../structIMB__MGR.html#a6e21e034cd011b695bac5a711be65da1',1,'IMB_MGR']]], + ['shifted_5fhkey_19',['shifted_hkey',['../structgcm__key__data.html#a1c73e3891c50c5e03e82715ceda77d17',1,'gcm_key_data']]], + ['shifted_5fhkey_5fk_20',['shifted_hkey_k',['../structgcm__key__data.html#a570a82f076ceb5da6719d1a554584820',1,'gcm_key_data']]], + ['sk16_21',['sk16',['../structkasumi__key__sched__s.html#a7e69de411b81753c10776442bbe03100',1,'kasumi_key_sched_s']]], + ['snow3g_5ff8_5f1_5fbuffer_22',['snow3g_f8_1_buffer',['../structIMB__MGR.html#ab70cbec09a61b5adeae37260c10234ce',1,'IMB_MGR']]], + ['snow3g_5ff8_5f1_5fbuffer_5fbit_23',['snow3g_f8_1_buffer_bit',['../structIMB__MGR.html#ad85fdcb101391b0a2488a3244d5f442d',1,'IMB_MGR']]], + ['snow3g_5ff8_5f1_5fbuffer_5fbit_5ft_24',['snow3g_f8_1_buffer_bit_t',['../intel-ipsec-mb_8h.html#a7bd00b31d12260305accf0d37b60c958',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f1_5fbuffer_5ft_25',['snow3g_f8_1_buffer_t',['../intel-ipsec-mb_8h.html#a281315fb8b441d837ebed541b3a23380',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f2_5fbuffer_26',['snow3g_f8_2_buffer',['../structIMB__MGR.html#a25f46509b2f8926902a73e2210d697fd',1,'IMB_MGR']]], + ['snow3g_5ff8_5f2_5fbuffer_5ft_27',['snow3g_f8_2_buffer_t',['../intel-ipsec-mb_8h.html#a4e8912e7e55a2ac891f85c19e7f29328',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f4_5fbuffer_28',['snow3g_f8_4_buffer',['../structIMB__MGR.html#a194a0f15a7345ee369780dbdeb4f52f6',1,'IMB_MGR']]], + ['snow3g_5ff8_5f4_5fbuffer_5ft_29',['snow3g_f8_4_buffer_t',['../intel-ipsec-mb_8h.html#ae13b1154dcc13ca562e81471659d6099',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f8_5fbuffer_30',['snow3g_f8_8_buffer',['../structIMB__MGR.html#a620d2d75bb50784aa0000b458197afaf',1,'IMB_MGR']]], + ['snow3g_5ff8_5f8_5fbuffer_5fmultikey_31',['snow3g_f8_8_buffer_multikey',['../structIMB__MGR.html#a898a4c56e1a746a328529c3665c5682e',1,'IMB_MGR']]], + ['snow3g_5ff8_5f8_5fbuffer_5fmultikey_5ft_32',['snow3g_f8_8_buffer_multikey_t',['../intel-ipsec-mb_8h.html#a109b2a72d05bbe05a373ed5cf3c1f201',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f8_5fbuffer_5ft_33',['snow3g_f8_8_buffer_t',['../intel-ipsec-mb_8h.html#a6155f472c6794e88289235752d83154a',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fiv_5fgen_34',['snow3g_f8_iv_gen',['../intel-ipsec-mb_8h.html#a2cbdb64c0b305cb5af010f9c98f70a43',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fn_5fbuffer_35',['snow3g_f8_n_buffer',['../structIMB__MGR.html#a829c4cf1157dfc3fcb829df31e408e9d',1,'IMB_MGR']]], + ['snow3g_5ff8_5fn_5fbuffer_5fmultikey_36',['snow3g_f8_n_buffer_multikey',['../structIMB__MGR.html#a63a2a5f74e7cc2fc637896bdaeb6b262',1,'IMB_MGR']]], + ['snow3g_5ff8_5fn_5fbuffer_5fmultikey_5ft_37',['snow3g_f8_n_buffer_multikey_t',['../intel-ipsec-mb_8h.html#a4049b601f9da6542e25d903efe524bc1',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fn_5fbuffer_5ft_38',['snow3g_f8_n_buffer_t',['../intel-ipsec-mb_8h.html#a6a4a1da950207c9ed04f3f18ae1b7e43',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff9_5f1_5fbuffer_39',['snow3g_f9_1_buffer',['../structIMB__MGR.html#a9a4d2148b6e8b4426b2946e9fe695461',1,'IMB_MGR']]], + ['snow3g_5ff9_5f1_5fbuffer_5ft_40',['snow3g_f9_1_buffer_t',['../intel-ipsec-mb_8h.html#ae7a48386f7190b4e46cd113eee413733',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff9_5fiv_5fgen_41',['snow3g_f9_iv_gen',['../intel-ipsec-mb_8h.html#a93e7a99927bc37fb4cb03f966fa4ebdd',1,'intel-ipsec-mb.h']]], + ['snow3g_5finit_5fkey_5fsched_42',['snow3g_init_key_sched',['../structIMB__MGR.html#a38ff0528815be4d92481948664366ca9',1,'IMB_MGR']]], + ['snow3g_5finit_5fkey_5fsched_5ft_43',['snow3g_init_key_sched_t',['../intel-ipsec-mb_8h.html#adfa182361f1c179d75129b4e46841af5',1,'intel-ipsec-mb.h']]], + ['snow3g_5fkey_5fsched_5fsize_44',['snow3g_key_sched_size',['../structIMB__MGR.html#a5e0002330399f643e8f0ca0c1b96a7a0',1,'IMB_MGR']]], + ['snow3g_5fkey_5fsched_5fsize_5ft_45',['snow3g_key_sched_size_t',['../intel-ipsec-mb_8h.html#ad78a4f1c0d6de808f7046c0fbcd98d8f',1,'intel-ipsec-mb.h']]], + ['snow3g_5fkey_5fschedule_5fs_46',['snow3g_key_schedule_s',['../structsnow3g__key__schedule__s.html',1,'']]], + ['snow3g_5fkey_5fschedule_5ft_47',['snow3g_key_schedule_t',['../intel-ipsec-mb_8h.html#ae50b48832b41b4b468cf933b44849213',1,'intel-ipsec-mb.h']]], + ['snow3g_5fuea2_5fooo_48',['snow3g_uea2_ooo',['../structIMB__MGR.html#ab0193bf46978026a82803c25e11acbfb',1,'IMB_MGR']]], + ['snow3g_5fuia2_49',['SNOW3G_UIA2',['../structIMB__JOB.html#a2a48f64b9344d0b6ca6775a4fdbe21b7',1,'IMB_JOB']]], + ['snow3g_5fuia2_5fooo_50',['snow3g_uia2_ooo',['../structIMB__MGR.html#aa7d4cfe53e0d70c22d3a274b8cd218af',1,'IMB_MGR']]], + ['snow_5fv_5faead_51',['SNOW_V_AEAD',['../structIMB__JOB.html#abf4a3f2db05c6583de5a99d9590ddca1',1,'IMB_JOB']]], + ['src_52',['src',['../structIMB__JOB.html#af83d63eb71a4f26b4915562bea554f28',1,'IMB_JOB']]], + ['sse_5favx_53',['sse_avx',['../structgcm__key__data.html#a908988e25607cd3e092bfcea9964deb1',1,'gcm_key_data']]], + ['status_54',['status',['../structIMB__JOB.html#ad4e08611eee5188cb84cb2dc337ef9f0',1,'IMB_JOB']]], + ['submit_5fburst_55',['submit_burst',['../structIMB__MGR.html#aa69002f8d8e17c0048d7aff9b3a1894e',1,'IMB_MGR']]], + ['submit_5fburst_5fnocheck_56',['submit_burst_nocheck',['../structIMB__MGR.html#a34adc55c8041e211c73c632628cd60d3',1,'IMB_MGR']]], + ['submit_5fcipher_5fburst_57',['submit_cipher_burst',['../structIMB__MGR.html#a8b52f1b3ca92eff48b945929a0eab839',1,'IMB_MGR']]], + ['submit_5fcipher_5fburst_5fnocheck_58',['submit_cipher_burst_nocheck',['../structIMB__MGR.html#acb2ee84d30247bb257635063db3ae308',1,'IMB_MGR']]], + ['submit_5fcipher_5fburst_5ft_59',['submit_cipher_burst_t',['../intel-ipsec-mb_8h.html#ac4bd89ab3cb1341eb4f30c4c501a9009',1,'intel-ipsec-mb.h']]], + ['submit_5fhash_5fburst_60',['submit_hash_burst',['../structIMB__MGR.html#a20a3d4d1e40aed26d83db5395b7ccc83',1,'IMB_MGR']]], + ['submit_5fhash_5fburst_5fnocheck_61',['submit_hash_burst_nocheck',['../structIMB__MGR.html#a809782eb9dcbd264442bcd9edd2f4c2f',1,'IMB_MGR']]], + ['submit_5fhash_5fburst_5ft_62',['submit_hash_burst_t',['../intel-ipsec-mb_8h.html#a4f6fca648737cec9ee869786e9c240b5',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_63',['submit_job',['../structIMB__MGR.html#ab143534d96e0d585b19dfc19553e929b',1,'IMB_MGR']]], + ['submit_5fjob_5favx_64',['submit_job_avx',['../intel-ipsec-mb_8h.html#a950bf3b7a3c63a371b561ea20aaf3a18',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx2_65',['submit_job_avx2',['../intel-ipsec-mb_8h.html#a8913995859e45005f8de57a3456c93cf',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx512_66',['submit_job_avx512',['../intel-ipsec-mb_8h.html#a75cc0d090470e47cd8be04451ee31a68',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_67',['submit_job_nocheck',['../structIMB__MGR.html#aab7350eeecc56b1ef45606b83519a72c',1,'IMB_MGR']]], + ['submit_5fjob_5fnocheck_5favx_68',['submit_job_nocheck_avx',['../intel-ipsec-mb_8h.html#ac7b339426753d3a7c44625a5ded3170a',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx2_69',['submit_job_nocheck_avx2',['../intel-ipsec-mb_8h.html#a72caaf8162ae6f802f7c1fe570afe42e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx512_70',['submit_job_nocheck_avx512',['../intel-ipsec-mb_8h.html#a5a95b9268508d68b9027df98167d22cd',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5fsse_71',['submit_job_nocheck_sse',['../intel-ipsec-mb_8h.html#a35e4c8a3d2eba8a612f021572621f98e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fsse_72',['submit_job_sse',['../intel-ipsec-mb_8h.html#a7700e90058a74980bfd7d462a19d32a0',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5ft_73',['submit_job_t',['../intel-ipsec-mb_8h.html#a41297d789b434157e746e338f0af96ec',1,'intel-ipsec-mb.h']]], + ['suite_5fid_74',['suite_id',['../structIMB__JOB.html#a2e8f1fefae0b0fe245eb20e5cc2aa830',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_14.js b/docs/search/all_14.js new file mode 100644 index 0000000000000000000000000000000000000000..6cf3c83e8a91fd10b297de576e1f7b5ad0ca0392 --- /dev/null +++ b/docs/search/all_14.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['u_0',['u',['../structIMB__JOB.html#a5e6d8702069bf88ce81cd750df66bcbc',1,'IMB_JOB']]], + ['used_5farch_1',['used_arch',['../structIMB__MGR.html#a44b6c95f8ce611828b6acacdb459cdc9',1,'IMB_MGR']]], + ['user_5fdata_2',['user_data',['../structIMB__JOB.html#abd56907c9485830817f7b30870726aeb',1,'IMB_JOB']]], + ['user_5fdata2_3',['user_data2',['../structIMB__JOB.html#a4e68c8cf14552db625e26477650dfb8b',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_15.js b/docs/search/all_15.js new file mode 100644 index 0000000000000000000000000000000000000000..835924a736105db12f45df3c24015475afa9e269 --- /dev/null +++ b/docs/search/all_15.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['vaes_5favx512_0',['vaes_avx512',['../structgcm__key__data.html#a7ad44e105b23e41b7f79530b6370e509',1,'gcm_key_data']]] +]; diff --git a/docs/search/all_16.js b/docs/search/all_16.js new file mode 100644 index 0000000000000000000000000000000000000000..63d1f09e5709a32070586a1940783806982f69fa --- /dev/null +++ b/docs/search/all_16.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['xcbc_0',['XCBC',['../structIMB__JOB.html#a3508ac7d49477dffb93b6b9cfbf0ebf3',1,'IMB_JOB']]], + ['xcbc_5fkeyexp_1',['xcbc_keyexp',['../structIMB__MGR.html#ae575c2350f5fca9a182c541795bb7d4d',1,'IMB_MGR']]], + ['xcbc_5fkeyexp_5ft_2',['xcbc_keyexp_t',['../intel-ipsec-mb_8h.html#abb36f71b9fb4d928c2ad9fa462b80ae6',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_17.js b/docs/search/all_17.js new file mode 100644 index 0000000000000000000000000000000000000000..6057aa1ca26d93ad870810a3830845b28a039af4 --- /dev/null +++ b/docs/search/all_17.js @@ -0,0 +1,15 @@ +var searchData= +[ + ['zuc256_5feea3_5fooo_0',['zuc256_eea3_ooo',['../structIMB__MGR.html#a6d05b0a20196be514182920710dce2a8',1,'IMB_MGR']]], + ['zuc256_5feia3_5fooo_1',['zuc256_eia3_ooo',['../structIMB__MGR.html#a5f8744a8d64273d3f4cbcd9ad41ce3d5',1,'IMB_MGR']]], + ['zuc_5feea3_5f1_5fbuffer_5ft_2',['zuc_eea3_1_buffer_t',['../intel-ipsec-mb_8h.html#a0a3032f0698c42bd486d053e90c44fdd',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5f4_5fbuffer_5ft_3',['zuc_eea3_4_buffer_t',['../intel-ipsec-mb_8h.html#a7df2e3f063333d677ef7668626126c4d',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5fiv_5fgen_4',['zuc_eea3_iv_gen',['../intel-ipsec-mb_8h.html#a4931dd6b1b91456bfe885e28ada0aaf3',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5fn_5fbuffer_5ft_5',['zuc_eea3_n_buffer_t',['../intel-ipsec-mb_8h.html#a303f2dd11b1193938596b4ec69edf8cc',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5fooo_6',['zuc_eea3_ooo',['../structIMB__MGR.html#a26d22c5ff963b9c7ffc8af19292bce5c',1,'IMB_MGR']]], + ['zuc_5feia3_7',['ZUC_EIA3',['../structIMB__JOB.html#ab42f79355b9173c63aca9b0b577a2b61',1,'IMB_JOB']]], + ['zuc_5feia3_5f1_5fbuffer_5ft_8',['zuc_eia3_1_buffer_t',['../intel-ipsec-mb_8h.html#a95ed5580686f708ba2313179cd6064a8',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fiv_5fgen_9',['zuc_eia3_iv_gen',['../intel-ipsec-mb_8h.html#ae3cefd0e992f1390110f0ef9a3aa510f',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fn_5fbuffer_5ft_10',['zuc_eia3_n_buffer_t',['../intel-ipsec-mb_8h.html#ada45082bb04dd727c0deec5c4fc96fed',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fooo_11',['zuc_eia3_ooo',['../structIMB__MGR.html#a910f747aa1a0e6a46f87b0dfd2189219',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_2.js b/docs/search/all_2.js new file mode 100644 index 0000000000000000000000000000000000000000..02ae4a442acef79e3f52f24ff6b05b61f4b30e13 --- /dev/null +++ b/docs/search/all_2.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['burst_5ffn_5ft_0',['burst_fn_t',['../intel-ipsec-mb_8h.html#aaeed8e5ec28aba6b8deef269b7c4bb28',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_3.js b/docs/search/all_3.js new file mode 100644 index 0000000000000000000000000000000000000000..a99ea05db91a99e8bf49d30e9e929fc938a29f72 --- /dev/null +++ b/docs/search/all_3.js @@ -0,0 +1,41 @@ +var searchData= +[ + ['cbcs_0',['CBCS',['../structIMB__JOB.html#a832c73263792bf8895cf028035730c4e',1,'IMB_JOB']]], + ['ccm_1',['CCM',['../structIMB__JOB.html#ad43a9c9149ec16b8e9164771ec38d379',1,'IMB_JOB']]], + ['chacha20_5fpoly1305_2',['CHACHA20_POLY1305',['../structIMB__JOB.html#ace537bcd7b871fddd4514b7fd4f678a3',1,'IMB_JOB']]], + ['chacha20_5fpoly1305_5fcontext_5fdata_3',['chacha20_poly1305_context_data',['../structchacha20__poly1305__context__data.html',1,'']]], + ['chacha20_5fpoly1305_5fdec_5fupdate_4',['chacha20_poly1305_dec_update',['../structIMB__MGR.html#a715aede4176564ac475e42c8a377bd8d',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5fenc_5fupdate_5',['chacha20_poly1305_enc_update',['../structIMB__MGR.html#ab2c87c40f9844795e3987536c602ce03',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5ffinalize_6',['chacha20_poly1305_finalize',['../structIMB__MGR.html#ab18b3fcba2aa252bb31f66093bc62abb',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5finit_7',['chacha20_poly1305_init',['../structIMB__MGR.html#a67b96b1b26fa25391f9f4b83a7a4b6c0',1,'IMB_MGR']]], + ['chacha_5fpoly_5fenc_5fdec_5fupdate_5ft_8',['chacha_poly_enc_dec_update_t',['../intel-ipsec-mb_8h.html#a437de868b8dba4f9b878d99a82f80548',1,'intel-ipsec-mb.h']]], + ['chacha_5fpoly_5ffinalize_5ft_9',['chacha_poly_finalize_t',['../intel-ipsec-mb_8h.html#a1a96f063bf1bd07de158985b57cff4a0',1,'intel-ipsec-mb.h']]], + ['chacha_5fpoly_5finit_5ft_10',['chacha_poly_init_t',['../intel-ipsec-mb_8h.html#a4b0b9ffa331f8d675d24911639f0374f',1,'intel-ipsec-mb.h']]], + ['chain_5forder_11',['chain_order',['../structIMB__JOB.html#a41e9fa248ebb040ce1e074a2444b55d2',1,'IMB_JOB']]], + ['cipher_5fdirection_12',['cipher_direction',['../structIMB__JOB.html#a1449c51b88ccdb2f855525b89c05eb84',1,'IMB_JOB']]], + ['cipher_5ffields_13',['cipher_fields',['../structIMB__JOB.html#a006bf024dd3e809fad23c3df813c6fb2',1,'IMB_JOB']]], + ['cipher_5ffunc_14',['cipher_func',['../structIMB__JOB.html#a652eb3eee14e59be703bf28e69ca0671',1,'IMB_JOB']]], + ['cipher_5fmode_15',['cipher_mode',['../structIMB__JOB.html#a78113e875fdb681e775fd32ee67c7597',1,'IMB_JOB']]], + ['cipher_5fstart_5foffset_5fin_5fbits_16',['cipher_start_offset_in_bits',['../structIMB__JOB.html#a60a01da0056367a6f0a326265e4da4fb',1,'IMB_JOB']]], + ['cipher_5fstart_5fsrc_5foffset_5fin_5fbits_17',['cipher_start_src_offset_in_bits',['../structIMB__JOB.html#af5c226d1c875c4e30a8f7bd7c03f05b1',1,'IMB_JOB']]], + ['cipher_5fstart_5fsrc_5foffset_5fin_5fbytes_18',['cipher_start_src_offset_in_bytes',['../structIMB__JOB.html#adda0b9bd22d445402bb9866e423707c4',1,'IMB_JOB']]], + ['cmac_19',['CMAC',['../structIMB__JOB.html#a499c1e90c9004ab7bb50fac98b6a9048',1,'IMB_JOB']]], + ['cmac_5fsubkey_5fgen_5f128_20',['cmac_subkey_gen_128',['../structIMB__MGR.html#aa3404b7811d57821ea574994e0c88995',1,'IMB_MGR']]], + ['cmac_5fsubkey_5fgen_5f256_21',['cmac_subkey_gen_256',['../structIMB__MGR.html#abd500d2cc03f6470ac6988785bebdf76',1,'IMB_MGR']]], + ['cmac_5fsubkey_5fgen_5ft_22',['cmac_subkey_gen_t',['../intel-ipsec-mb_8h.html#aab07996d622d682adc1db8286e633cb3',1,'intel-ipsec-mb.h']]], + ['crc10_5fiuup_5fdata_23',['crc10_iuup_data',['../structIMB__MGR.html#a7d0979deda4e2ff7ac202c82611ce94c',1,'IMB_MGR']]], + ['crc11_5ffp_5fheader_24',['crc11_fp_header',['../structIMB__MGR.html#a4b901916c2f262f1c873e4cec7b7f474',1,'IMB_MGR']]], + ['crc16_5ffp_5fdata_25',['crc16_fp_data',['../structIMB__MGR.html#ad566ac43002c2ba8ecb93048d5edc41b',1,'IMB_MGR']]], + ['crc16_5fx25_26',['crc16_x25',['../structIMB__MGR.html#a52a7ab10d6434820e9b31d0e772ea0bc',1,'IMB_MGR']]], + ['crc24_5flte_5fa_27',['crc24_lte_a',['../structIMB__MGR.html#a058cfc60f2fdd68678b36f4f038270ac',1,'IMB_MGR']]], + ['crc24_5flte_5fb_28',['crc24_lte_b',['../structIMB__MGR.html#abe8406f3582d3540edb770e153b74c3e',1,'IMB_MGR']]], + ['crc32_5fethernet_5ffcs_29',['crc32_ethernet_fcs',['../structIMB__MGR.html#a8483109fbe42f8b7fa6e7f53e016d5ec',1,'IMB_MGR']]], + ['crc32_5ffn_5ft_30',['crc32_fn_t',['../intel-ipsec-mb_8h.html#a56dd197c301ad27dc9ec0862721cb73d',1,'intel-ipsec-mb.h']]], + ['crc32_5fsctp_31',['crc32_sctp',['../structIMB__MGR.html#a4aed66c9e73808a65a062e1a5b0f66aa',1,'IMB_MGR']]], + ['crc32_5fwimax_5fofdma_5fdata_32',['crc32_wimax_ofdma_data',['../structIMB__MGR.html#ad08e44bc4edc54c556ea2245f848b974',1,'IMB_MGR']]], + ['crc6_5fiuup_5fheader_33',['crc6_iuup_header',['../structIMB__MGR.html#aa610dc7226673020be410762b995b725',1,'IMB_MGR']]], + ['crc7_5ffp_5fheader_34',['crc7_fp_header',['../structIMB__MGR.html#ac819ddcc968604186983d58f275bd24b',1,'IMB_MGR']]], + ['crc8_5fwimax_5fofdma_5fhcs_35',['crc8_wimax_ofdma_hcs',['../structIMB__MGR.html#a5e5be572c46ed01ac11fc95738373e2a',1,'IMB_MGR']]], + ['ctx_36',['ctx',['../structIMB__JOB.html#a053ceeb65e611c48a97a3e10d03d47eb',1,'IMB_JOB::ctx'],['../structIMB__JOB.html#a4abdf3311d861979dc39669b3b7f1ed8',1,'IMB_JOB::ctx']]], + ['current_5fcounter_37',['current_counter',['../structgcm__context__data.html#a2a454a5e44422aaec135dc7807aa55a4',1,'gcm_context_data']]] +]; diff --git a/docs/search/all_4.js b/docs/search/all_4.js new file mode 100644 index 0000000000000000000000000000000000000000..992a8a9742190b5156c41edc85347274bbe1f806 --- /dev/null +++ b/docs/search/all_4.js @@ -0,0 +1,20 @@ +var searchData= +[ + ['dec_5fkeys_0',['dec_keys',['../structIMB__JOB.html#af78a3df5a9f79f329c8d1095b9ecea05',1,'IMB_JOB']]], + ['declare_5faligned_1',['DECLARE_ALIGNED',['../intel-ipsec-mb_8h.html#ad920a47ed8f57af8b2cf6bcd18a3e41c',1,'intel-ipsec-mb.h']]], + ['des3_5fdec_5fooo_2',['des3_dec_ooo',['../structIMB__MGR.html#a2ab69fdbef6a9a0574fcbb5ea9a7da3b',1,'IMB_MGR']]], + ['des3_5fenc_5fooo_3',['des3_enc_ooo',['../structIMB__MGR.html#a87f7566634911b4dba48a19a0c3a4993',1,'IMB_MGR']]], + ['des_5fcfb_5fone_4',['des_cfb_one',['../intel-ipsec-mb_8h.html#a71a0353411f5307a2177ae9b2814f522',1,'intel-ipsec-mb.h']]], + ['des_5fdec_5fooo_5',['des_dec_ooo',['../structIMB__MGR.html#af03e401b002e2e415c8a927c46e1517b',1,'IMB_MGR']]], + ['des_5fenc_5fooo_6',['des_enc_ooo',['../structIMB__MGR.html#a54ec50411917f1aa9016db268ac638c1',1,'IMB_MGR']]], + ['des_5fkey_5fsched_7',['des_key_sched',['../structIMB__MGR.html#aae9afac5c2c1bf7b27d187d10778a1ee',1,'IMB_MGR']]], + ['des_5fkey_5fschedule_8',['des_key_schedule',['../intel-ipsec-mb_8h.html#a9911462fe4689c7420b827bb13fc3440',1,'intel-ipsec-mb.h']]], + ['des_5fkeysched_5ft_9',['des_keysched_t',['../intel-ipsec-mb_8h.html#a60a93569304a978c798bb8c4a4f7f52f',1,'intel-ipsec-mb.h']]], + ['docsis128_5fcrc32_5fsec_5fooo_10',['docsis128_crc32_sec_ooo',['../structIMB__MGR.html#acf438b50cb95a4fff1276b6666d22ad4',1,'IMB_MGR']]], + ['docsis128_5fsec_5fooo_11',['docsis128_sec_ooo',['../structIMB__MGR.html#a93cb5904608e179c49917f80c64fabd1',1,'IMB_MGR']]], + ['docsis256_5fcrc32_5fsec_5fooo_12',['docsis256_crc32_sec_ooo',['../structIMB__MGR.html#af6531598b8dd0131a11e0cf95854605d',1,'IMB_MGR']]], + ['docsis256_5fsec_5fooo_13',['docsis256_sec_ooo',['../structIMB__MGR.html#acb6e3b70e16194274e2eaefdff8f9e89',1,'IMB_MGR']]], + ['docsis_5fdes_5fdec_5fooo_14',['docsis_des_dec_ooo',['../structIMB__MGR.html#a3d5173cea3d4932576f0a5b3abbc1f40',1,'IMB_MGR']]], + ['docsis_5fdes_5fenc_5fooo_15',['docsis_des_enc_ooo',['../structIMB__MGR.html#a937fb0d3b113d203ee1a4beaf5b5b98f',1,'IMB_MGR']]], + ['dst_16',['dst',['../structIMB__JOB.html#ac1f7c363e7709cf1a7de1364aa8625e2',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_5.js b/docs/search/all_5.js new file mode 100644 index 0000000000000000000000000000000000000000..086cbee17698e4755ab97cb0afa13be3a6772917 --- /dev/null +++ b/docs/search/all_5.js @@ -0,0 +1,12 @@ +var searchData= +[ + ['earliest_5fjob_0',['earliest_job',['../structIMB__MGR.html#ac5cd33ac3bf985dc31ccc716fb3a7747',1,'IMB_MGR']]], + ['eea3_5f1_5fbuffer_1',['eea3_1_buffer',['../structIMB__MGR.html#aeff54b370fbe495ce96370811d280140',1,'IMB_MGR']]], + ['eea3_5f4_5fbuffer_2',['eea3_4_buffer',['../structIMB__MGR.html#ac5ad1bf8ecdcfc51eaf681fb353c9621',1,'IMB_MGR']]], + ['eea3_5fn_5fbuffer_3',['eea3_n_buffer',['../structIMB__MGR.html#a67d8a55ba87742b8cf282d23ed9e3ef5',1,'IMB_MGR']]], + ['eia3_5f1_5fbuffer_4',['eia3_1_buffer',['../structIMB__MGR.html#a3ed58bdf37ab6d5f0d28271882d52327',1,'IMB_MGR']]], + ['eia3_5fn_5fbuffer_5',['eia3_n_buffer',['../structIMB__MGR.html#ab515c5aec39b1a9a31454b945e5ccd35',1,'IMB_MGR']]], + ['enc_5fkeys_6',['enc_keys',['../structIMB__JOB.html#ac656e2d98f36bd5547cc9fa297c1de62',1,'IMB_JOB']]], + ['end_5fooo_7',['end_ooo',['../structIMB__MGR.html#a6e1ab772182b97f931af3d4685785353',1,'IMB_MGR']]], + ['expanded_5fkeys_8',['expanded_keys',['../structgcm__key__data.html#aa5bda4f55b8393be151cfe28a9dc1ca9',1,'gcm_key_data']]] +]; diff --git a/docs/search/all_6.js b/docs/search/all_6.js new file mode 100644 index 0000000000000000000000000000000000000000..c95af44d244d6671c9aba2063a61bab644bf3f14 --- /dev/null +++ b/docs/search/all_6.js @@ -0,0 +1,21 @@ +var searchData= +[ + ['f8_5f1_5fbuffer_0',['f8_1_buffer',['../structIMB__MGR.html#af854e664cef84cb8be4495a1af5e9cb5',1,'IMB_MGR']]], + ['f8_5f1_5fbuffer_5fbit_1',['f8_1_buffer_bit',['../structIMB__MGR.html#aacf6ce3f508fef115dcd2dc415ca7379',1,'IMB_MGR']]], + ['f8_5f2_5fbuffer_2',['f8_2_buffer',['../structIMB__MGR.html#ac733fc4c5ee6e80e8e71b20a9bb2dc6e',1,'IMB_MGR']]], + ['f8_5f3_5fbuffer_3',['f8_3_buffer',['../structIMB__MGR.html#a929b9c50bc98bf37daec7a1603f10118',1,'IMB_MGR']]], + ['f8_5f4_5fbuffer_4',['f8_4_buffer',['../structIMB__MGR.html#af0b1c5d3b269b252a75a55df8bdf472c',1,'IMB_MGR']]], + ['f8_5fn_5fbuffer_5',['f8_n_buffer',['../structIMB__MGR.html#ae0f2316d96e46b64d08e3cf73a93b39d',1,'IMB_MGR']]], + ['f9_5f1_5fbuffer_6',['f9_1_buffer',['../structIMB__MGR.html#afafb95a00fe6633fb6901855928b08bb',1,'IMB_MGR']]], + ['f9_5f1_5fbuffer_5fuser_7',['f9_1_buffer_user',['../structIMB__MGR.html#a4fbdac066c9d0fbd1d586a2e5b2319c3',1,'IMB_MGR']]], + ['features_8',['features',['../structIMB__MGR.html#af877c7669265775604857c6b81063bdb',1,'IMB_MGR']]], + ['flags_9',['flags',['../structIMB__MGR.html#ae1a596c9c9d6ca1292c6116575021c14',1,'IMB_MGR']]], + ['flush_5fburst_10',['flush_burst',['../structIMB__MGR.html#aa0283df1a5cd7a6dfc78c8f9896ea3ec',1,'IMB_MGR']]], + ['flush_5fjob_11',['flush_job',['../structIMB__MGR.html#a225e3b35eef8e18fd09c429fbb2f70b5',1,'IMB_MGR']]], + ['flush_5fjob_5favx_12',['flush_job_avx',['../intel-ipsec-mb_8h.html#abc6b0f1c857f26b834f22b6bd8f51846',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5favx2_13',['flush_job_avx2',['../intel-ipsec-mb_8h.html#ae242d142065172898129f30e6bc40d8c',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5favx512_14',['flush_job_avx512',['../intel-ipsec-mb_8h.html#ac7aec54f653aeee2554ba980d62fd491',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5fsse_15',['flush_job_sse',['../intel-ipsec-mb_8h.html#a96b880b8a32b0466f880d4f22d95cc3b',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5ft_16',['flush_job_t',['../intel-ipsec-mb_8h.html#ae64d0f677e42b8f8d50289113cf5e01e',1,'intel-ipsec-mb.h']]], + ['free_5fmb_5fmgr_17',['free_mb_mgr',['../intel-ipsec-mb_8h.html#a238b96cc36c0dc64798a1244b94add8b',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_7.js b/docs/search/all_7.js new file mode 100644 index 0000000000000000000000000000000000000000..53ee649d099397def220d14cd455fc9eca40436e --- /dev/null +++ b/docs/search/all_7.js @@ -0,0 +1,64 @@ +var searchData= +[ + ['gcm_0',['GCM',['../structIMB__JOB.html#ac9fc7aef831cfc1ca598a55b250dd858',1,'IMB_JOB']]], + ['gcm128_5fdec_1',['gcm128_dec',['../structIMB__MGR.html#a2ce2f6e6d61953b24a5859279534b02a',1,'IMB_MGR']]], + ['gcm128_5fdec_5ffinalize_2',['gcm128_dec_finalize',['../structIMB__MGR.html#af95fa831fb24629e19613e8594e5d32a',1,'IMB_MGR']]], + ['gcm128_5fdec_5fupdate_3',['gcm128_dec_update',['../structIMB__MGR.html#a28ddfed1a807fedb9c77c99736278dcb',1,'IMB_MGR']]], + ['gcm128_5fenc_4',['gcm128_enc',['../structIMB__MGR.html#a9f8b9687fbb17f7a78fc4ba79e2749fd',1,'IMB_MGR']]], + ['gcm128_5fenc_5ffinalize_5',['gcm128_enc_finalize',['../structIMB__MGR.html#a8efc0cf95907805289a297c024ce0cac',1,'IMB_MGR']]], + ['gcm128_5fenc_5fupdate_6',['gcm128_enc_update',['../structIMB__MGR.html#ad2c64d9d5f9ffd65d13e73205cb803cd',1,'IMB_MGR']]], + ['gcm128_5finit_7',['gcm128_init',['../structIMB__MGR.html#ad8c004899d29974a3f4c034370422c31',1,'IMB_MGR']]], + ['gcm128_5finit_5fvar_5fiv_8',['gcm128_init_var_iv',['../structIMB__MGR.html#aebd9eab76cd056add559e6bea328dbf0',1,'IMB_MGR']]], + ['gcm128_5fpre_9',['gcm128_pre',['../structIMB__MGR.html#a207c78e01e4aa1a35de1bbd2cc0a6ee2',1,'IMB_MGR']]], + ['gcm128_5fprecomp_10',['gcm128_precomp',['../structIMB__MGR.html#a69e4dd044703cd3aa9c4c36e04b2a701',1,'IMB_MGR']]], + ['gcm192_5fdec_11',['gcm192_dec',['../structIMB__MGR.html#adff2cf46e7cbb5e2926f348623d8ad2a',1,'IMB_MGR']]], + ['gcm192_5fdec_5ffinalize_12',['gcm192_dec_finalize',['../structIMB__MGR.html#a4d7879172344be144b92be68b2e2a81a',1,'IMB_MGR']]], + ['gcm192_5fdec_5fupdate_13',['gcm192_dec_update',['../structIMB__MGR.html#a66fa94f9c07061f3d93b78ed505931e3',1,'IMB_MGR']]], + ['gcm192_5fenc_14',['gcm192_enc',['../structIMB__MGR.html#aaaeb1676e0cf19e3558d10e537f71bda',1,'IMB_MGR']]], + ['gcm192_5fenc_5ffinalize_15',['gcm192_enc_finalize',['../structIMB__MGR.html#a91ad9ce7a4b3c9ceeffa9dd712013915',1,'IMB_MGR']]], + ['gcm192_5fenc_5fupdate_16',['gcm192_enc_update',['../structIMB__MGR.html#a3b5c3407cc4595420dbe43e98090e558',1,'IMB_MGR']]], + ['gcm192_5finit_17',['gcm192_init',['../structIMB__MGR.html#a006315a7045996d451c7dfd03b96a076',1,'IMB_MGR']]], + ['gcm192_5finit_5fvar_5fiv_18',['gcm192_init_var_iv',['../structIMB__MGR.html#a7bfa7b17c0fb36dc832138a86948991f',1,'IMB_MGR']]], + ['gcm192_5fpre_19',['gcm192_pre',['../structIMB__MGR.html#a149b463ecf68ee78d39c33e183ed8997',1,'IMB_MGR']]], + ['gcm192_5fprecomp_20',['gcm192_precomp',['../structIMB__MGR.html#abec8bdb319ca132028d5b787bf60a895',1,'IMB_MGR']]], + ['gcm256_5fdec_21',['gcm256_dec',['../structIMB__MGR.html#acfdfb511771cac71e032c5bf1c9273b7',1,'IMB_MGR']]], + ['gcm256_5fdec_5ffinalize_22',['gcm256_dec_finalize',['../structIMB__MGR.html#a5c2e4a1296e33f5c9eaff2067a6432d4',1,'IMB_MGR']]], + ['gcm256_5fdec_5fupdate_23',['gcm256_dec_update',['../structIMB__MGR.html#a99fcb1d18fa45044063c6c288b0fc769',1,'IMB_MGR']]], + ['gcm256_5fenc_24',['gcm256_enc',['../structIMB__MGR.html#a79767326692a8d4faabe5ab435dc5df8',1,'IMB_MGR']]], + ['gcm256_5fenc_5ffinalize_25',['gcm256_enc_finalize',['../structIMB__MGR.html#a9764d27efef2491a86d517f38dd9b6ef',1,'IMB_MGR']]], + ['gcm256_5fenc_5fupdate_26',['gcm256_enc_update',['../structIMB__MGR.html#ac131ceec0dcb0d08097b76c8abe9e024',1,'IMB_MGR']]], + ['gcm256_5finit_27',['gcm256_init',['../structIMB__MGR.html#a8987d7d491628d9d80f12820468ac541',1,'IMB_MGR']]], + ['gcm256_5finit_5fvar_5fiv_28',['gcm256_init_var_iv',['../structIMB__MGR.html#afa275308dab467d0c68dc921bd68d1a4',1,'IMB_MGR']]], + ['gcm256_5fpre_29',['gcm256_pre',['../structIMB__MGR.html#a44b0b9d218043be965a84c29ced2584b',1,'IMB_MGR']]], + ['gcm256_5fprecomp_30',['gcm256_precomp',['../structIMB__MGR.html#a1ea15abc11b618ceb438c7f282643c52',1,'IMB_MGR']]], + ['gcm_5fcontext_5fdata_31',['gcm_context_data',['../structgcm__context__data.html',1,'']]], + ['gcm_5fkey_5fdata_32',['gcm_key_data',['../structgcm__key__data.html',1,'']]], + ['get_5fcompleted_5fjob_33',['get_completed_job',['../structIMB__MGR.html#a99b7111c8fc59161e0bf291d13b1f195',1,'IMB_MGR']]], + ['get_5fcompleted_5fjob_5favx_34',['get_completed_job_avx',['../intel-ipsec-mb_8h.html#a9d115c47670330bd01b960f2188169d9',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5favx2_35',['get_completed_job_avx2',['../intel-ipsec-mb_8h.html#ad85bdbf56038fd72ceda7e50deea586e',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5favx512_36',['get_completed_job_avx512',['../intel-ipsec-mb_8h.html#a693afd528fcc0809cc667b6c38de84dc',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5fsse_37',['get_completed_job_sse',['../intel-ipsec-mb_8h.html#a267a718472e0c1b6585402e81e44e7e9',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5ft_38',['get_completed_job_t',['../intel-ipsec-mb_8h.html#ac8cb9a545cdef14f60f36fe8888e6e7c',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fburst_39',['get_next_burst',['../structIMB__MGR.html#a8c6b03b886ce7706a6cc8319345e8acf',1,'IMB_MGR']]], + ['get_5fnext_5fjob_40',['get_next_job',['../structIMB__MGR.html#a35939892cd168eb7ce9c6c29c5401709',1,'IMB_MGR']]], + ['get_5fnext_5fjob_5favx_41',['get_next_job_avx',['../intel-ipsec-mb_8h.html#a3518b02f06a82e9f83cd0435f3cf25c9',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5favx2_42',['get_next_job_avx2',['../intel-ipsec-mb_8h.html#a49a3d54979188b3c68bdd621043125f7',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5favx512_43',['get_next_job_avx512',['../intel-ipsec-mb_8h.html#a40c08ba2c387e24f2096b1213c46c812',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5fsse_44',['get_next_job_sse',['../intel-ipsec-mb_8h.html#a055db1f6f16bc17724bdc64d0f313871',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5ft_45',['get_next_job_t',['../intel-ipsec-mb_8h.html#a5a533d852e3b65e9e5978fec37da43b5',1,'intel-ipsec-mb.h']]], + ['ghash_46',['GHASH',['../structIMB__JOB.html#af24c379b5414a5d3ec290954b98b1c72',1,'IMB_JOB']]], + ['ghash_47',['ghash',['../structIMB__MGR.html#afc7d7ae10e8e6f37b9ec894059818fad',1,'IMB_MGR']]], + ['ghash_5fkeys_48',['ghash_keys',['../structgcm__key__data.html#af5adb4225d569050207cfcfe00fe273a',1,'gcm_key_data']]], + ['ghash_5fpre_49',['ghash_pre',['../structIMB__MGR.html#af6a1c323981b8c3cfe09f94cd9dd58e2',1,'IMB_MGR']]], + ['ghash_5ft_50',['ghash_t',['../intel-ipsec-mb_8h.html#a12e0c05f0ecec8d75d4b288a7ec11c84',1,'intel-ipsec-mb.h']]], + ['gmac_51',['GMAC',['../structIMB__JOB.html#a07832e4571ec56f377dffa477b7aafe0',1,'IMB_JOB']]], + ['gmac128_5ffinalize_52',['gmac128_finalize',['../structIMB__MGR.html#a00ce9a8422a961166ce9855d4815cc2e',1,'IMB_MGR']]], + ['gmac128_5finit_53',['gmac128_init',['../structIMB__MGR.html#a1098c38784f6c6e4e3a3380f8be8ecc1',1,'IMB_MGR']]], + ['gmac128_5fupdate_54',['gmac128_update',['../structIMB__MGR.html#a90aad4f47448a1959b7cf747438d7ce9',1,'IMB_MGR']]], + ['gmac192_5ffinalize_55',['gmac192_finalize',['../structIMB__MGR.html#a000b24d7fbb9b57ed6bd4147d225e015',1,'IMB_MGR']]], + ['gmac192_5finit_56',['gmac192_init',['../structIMB__MGR.html#a3dd58a9d9b4fb6cf85fa26175ffac9a6',1,'IMB_MGR']]], + ['gmac192_5fupdate_57',['gmac192_update',['../structIMB__MGR.html#a5220f0d243291ae5a2f43fbc93d18725',1,'IMB_MGR']]], + ['gmac256_5ffinalize_58',['gmac256_finalize',['../structIMB__MGR.html#a3bb98c9a9c8545f363b74627870c7038',1,'IMB_MGR']]], + ['gmac256_5finit_59',['gmac256_init',['../structIMB__MGR.html#aca7d6388863eb76b60651e7b16c41b82',1,'IMB_MGR']]], + ['gmac256_5fupdate_60',['gmac256_update',['../structIMB__MGR.html#a0720a1fc9c0652c6b3aa7dfab070f73d',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_8.js b/docs/search/all_8.js new file mode 100644 index 0000000000000000000000000000000000000000..21cb71d476c937544568ac1a3d4ab693d1446629 --- /dev/null +++ b/docs/search/all_8.js @@ -0,0 +1,22 @@ +var searchData= +[ + ['hash_0',['hash',['../structchacha20__poly1305__context__data.html#ae12447b2fd2e9f8a6f09f443bd303887',1,'chacha20_poly1305_context_data']]], + ['hash_5falg_1',['hash_alg',['../structIMB__JOB.html#aa4d9b62d3353150aba88aec5d8d08853',1,'IMB_JOB']]], + ['hash_5ffn_5ft_2',['hash_fn_t',['../intel-ipsec-mb_8h.html#afacacc8b74c05c80a0ee255c68637490',1,'intel-ipsec-mb.h']]], + ['hash_5ffunc_3',['hash_func',['../structIMB__JOB.html#a62883bf8707d09816753832bef5d2902',1,'IMB_JOB']]], + ['hash_5flen_4',['hash_len',['../structchacha20__poly1305__context__data.html#ade4568d041695b318df773787872127f',1,'chacha20_poly1305_context_data']]], + ['hash_5fone_5fblock_5ft_5',['hash_one_block_t',['../intel-ipsec-mb_8h.html#aade58eb541a45650ed7d875dc2be97fa',1,'intel-ipsec-mb.h']]], + ['hash_5fstart_5fsrc_5foffset_5fin_5fbytes_6',['hash_start_src_offset_in_bytes',['../structIMB__JOB.html#af7cdc3d086f6e2b12d21c1383b2bd152',1,'IMB_JOB']]], + ['hec_5f32_7',['hec_32',['../structIMB__MGR.html#a0863c9f64126aba5cb47628493dae582',1,'IMB_MGR']]], + ['hec_5f32_5ft_8',['hec_32_t',['../intel-ipsec-mb_8h.html#aa34b11bf4be97c764b283aedc0589575',1,'intel-ipsec-mb.h']]], + ['hec_5f64_9',['hec_64',['../structIMB__MGR.html#a4128ee02f4cbef5aa1e6b0bb5fd46097',1,'IMB_MGR']]], + ['hec_5f64_5ft_10',['hec_64_t',['../intel-ipsec-mb_8h.html#aa9afd4ee91f0d82729afcf5f1cdcb808',1,'intel-ipsec-mb.h']]], + ['high_11',['high',['../structimb__uint128__t.html#aa65815c9b383e888758ae21d0c6159c9',1,'imb_uint128_t']]], + ['hmac_12',['HMAC',['../structIMB__JOB.html#aa84f5eefd378ab65ebe0b302ccb0df92',1,'IMB_JOB']]], + ['hmac_5fmd5_5fooo_13',['hmac_md5_ooo',['../structIMB__MGR.html#a855d14df536b22ceb175dc495ced891c',1,'IMB_MGR']]], + ['hmac_5fsha_5f1_5fooo_14',['hmac_sha_1_ooo',['../structIMB__MGR.html#a49be1d77b96d338c08200dce82e452d3',1,'IMB_MGR']]], + ['hmac_5fsha_5f224_5fooo_15',['hmac_sha_224_ooo',['../structIMB__MGR.html#ab9226cf37e41c2a13ae6003305654193',1,'IMB_MGR']]], + ['hmac_5fsha_5f256_5fooo_16',['hmac_sha_256_ooo',['../structIMB__MGR.html#a633807f7eaae35ada650a2c22f1366be',1,'IMB_MGR']]], + ['hmac_5fsha_5f384_5fooo_17',['hmac_sha_384_ooo',['../structIMB__MGR.html#afd72d6af1d8ff605e8aaaf984f0700ed',1,'IMB_MGR']]], + ['hmac_5fsha_5f512_5fooo_18',['hmac_sha_512_ooo',['../structIMB__MGR.html#af0cf9149495b0805a4f6f511c0fbff5b',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_9.js b/docs/search/all_9.js new file mode 100644 index 0000000000000000000000000000000000000000..5c9d55802b0d4f8efb368aeee6350c93581c217f --- /dev/null +++ b/docs/search/all_9.js @@ -0,0 +1,399 @@ +var searchData= +[ + ['imb_5faes128_5fcfb_5fone_0',['IMB_AES128_CFB_ONE',['../intel-ipsec-mb_8h.html#a1e122b460a714f2981f94bf7f4bba547',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_1',['IMB_AES128_GCM_DEC',['../intel-ipsec-mb_8h.html#a2b026b2a901c76a1512eb9c8c80f9709',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5ffinalize_2',['IMB_AES128_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a280b9231c76c525a62d0ef819abd2133',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5fupdate_3',['IMB_AES128_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a2f6014cfaaa0ee5f454e2771ff6b5829',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_4',['IMB_AES128_GCM_ENC',['../intel-ipsec-mb_8h.html#adb65fc01b63a475cca4cf532ace005b6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5ffinalize_5',['IMB_AES128_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#aaf13d46f5832cf9e0cb50f7fcde020be',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5fupdate_6',['IMB_AES128_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#ab1f4c52b4145946e80ef698e469d67db',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_7',['IMB_AES128_GCM_INIT',['../intel-ipsec-mb_8h.html#ad0a8ea6b81df94d066b028ca7bf656ef',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_5fvar_5fiv_8',['IMB_AES128_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a04142f11a678c0419dff242a0ccbd2e6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fpre_9',['IMB_AES128_GCM_PRE',['../intel-ipsec-mb_8h.html#a5a442c1fcd7d427bf78e804da32bdbd1',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fprecomp_10',['IMB_AES128_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a1c63dfec645afa671f93f4fa65a6996e',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5ffinalize_11',['IMB_AES128_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#af0bd4f5841d41433339717e6b97c15ad',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5finit_12',['IMB_AES128_GMAC_INIT',['../intel-ipsec-mb_8h.html#af315c30a8cbd3c20735e22f462b47964',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5fupdate_13',['IMB_AES128_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#af0b2ba1a0f9b099a06a1ba04380b0e2a',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_14',['IMB_AES192_GCM_DEC',['../intel-ipsec-mb_8h.html#aedd2b84ebfbcb56ac5176e8793490047',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5ffinalize_15',['IMB_AES192_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a759bbe1b8494c3157f1a506c003b700b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5fupdate_16',['IMB_AES192_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#ae25c4f893fc29f09d03a8ece5fddc82f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_17',['IMB_AES192_GCM_ENC',['../intel-ipsec-mb_8h.html#af7548f17077a0b187a4ecfde4e0ff5d9',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5ffinalize_18',['IMB_AES192_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1e36a361be137a5d49b0798f054403bd',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5fupdate_19',['IMB_AES192_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a456f5bed8eb67603bdebfda230f76957',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_20',['IMB_AES192_GCM_INIT',['../intel-ipsec-mb_8h.html#a8cd402fe539de3fc8ff4062c2804096f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_5fvar_5fiv_21',['IMB_AES192_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a97791c976660a673405e6fa22ef3d262',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fpre_22',['IMB_AES192_GCM_PRE',['../intel-ipsec-mb_8h.html#a003f16aac382baff7a6b83040e58e471',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fprecomp_23',['IMB_AES192_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#ac7b3bcfd3ecbe7b2b8c7da963310b87b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5ffinalize_24',['IMB_AES192_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#a16a6f89fd4131c406d6ee7a1aeb663eb',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5finit_25',['IMB_AES192_GMAC_INIT',['../intel-ipsec-mb_8h.html#a25ed49507e66a6e98eab4a3d34e75231',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5fupdate_26',['IMB_AES192_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a113d0f4320e929f89786773f7d11c5c8',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fcfb_5fone_27',['IMB_AES256_CFB_ONE',['../intel-ipsec-mb_8h.html#ab4d68ab67a7d4a471f5cb454301aac9d',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_28',['IMB_AES256_GCM_DEC',['../intel-ipsec-mb_8h.html#a1eb77771626f83eba8482473d29ffbb1',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5ffinalize_29',['IMB_AES256_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a64d9473bc4f28c0b783baa4a31b9930e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5fupdate_30',['IMB_AES256_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a70847fd3bb5d1be79ebfa4e9104edabd',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_31',['IMB_AES256_GCM_ENC',['../intel-ipsec-mb_8h.html#a556333acdc2db1a2540a1cd1af644ca9',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5ffinalize_32',['IMB_AES256_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1f49fe7109315eb479ac1c9969e2f877',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5fupdate_33',['IMB_AES256_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a223c00c636e2552e7a3dd1a97952cc6a',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_34',['IMB_AES256_GCM_INIT',['../intel-ipsec-mb_8h.html#a8049111340bf2694799eb4e42b0a82c3',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_5fvar_5fiv_35',['IMB_AES256_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a61ddc1485781aba1969fe3ca97a22ded',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fpre_36',['IMB_AES256_GCM_PRE',['../intel-ipsec-mb_8h.html#a5771f5169d3f3af3a2419ccd8275a754',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fprecomp_37',['IMB_AES256_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a4fa417a748ca30a961334bf91297dfda',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5ffinalize_38',['IMB_AES256_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#abfbb7469f7d0240bb3d9a0e5f368b19e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5finit_39',['IMB_AES256_GMAC_INIT',['../intel-ipsec-mb_8h.html#a3d57940a54e9d94fa14dc008cd959092',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5fupdate_40',['IMB_AES256_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a91b8d98c7ffce8179ef953493a959d4b',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fblock_5fsize_41',['IMB_AES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#ab1b5db5ac58714b6f69e754a0c4ca793',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f128_42',['IMB_AES_CMAC_SUBKEY_GEN_128',['../intel-ipsec-mb_8h.html#aa1e9006a8a76ba7fb09b97816d7fdfe5',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f256_43',['IMB_AES_CMAC_SUBKEY_GEN_256',['../intel-ipsec-mb_8h.html#aca7b22139c297858794c7268439e88f0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f128_44',['IMB_AES_KEYEXP_128',['../intel-ipsec-mb_8h.html#afa111339421e794c2643df2ecf86a8c0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f192_45',['IMB_AES_KEYEXP_192',['../intel-ipsec-mb_8h.html#a574d86849e2c9ca402305eb479ca558d',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f256_46',['IMB_AES_KEYEXP_256',['../intel-ipsec-mb_8h.html#a003306124dae4920161ed6d4a7bf4e2f',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fxcbc_5fkeyexp_47',['IMB_AES_XCBC_KEYEXP',['../intel-ipsec-mb_8h.html#a5a5f2342f2b1e65e41084d915c3b2cda',1,'intel-ipsec-mb.h']]], + ['imb_5farch_48',['IMB_ARCH',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx_49',['IMB_ARCH_AVX',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6aa831f565e051aa13fcad5b224358e57d',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx2_50',['IMB_ARCH_AVX2',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a701628fddfdd84d56495839f20573de6',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx512_51',['IMB_ARCH_AVX512',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1df4f12c5769b52a95f1a4b89a8163c8',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnoaesni_52',['IMB_ARCH_NOAESNI',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a6a5b29422391adbb6806575bf7ccae44',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnone_53',['IMB_ARCH_NONE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1ff5121849f183accf83fd652b854733',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnum_54',['IMB_ARCH_NUM',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6ae6536a65a8d0cd53faa325d5ee4dcdda',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fsse_55',['IMB_ARCH_SSE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a3c2ccbe898a62cbaed5bad7717fdfa16',1,'intel-ipsec-mb.h']]], + ['imb_5fassert_56',['IMB_ASSERT',['../intel-ipsec-mb_8h.html#af0f2f0bfad62f740bc2e22e60f48af8d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fccm_57',['IMB_AUTH_AES_CCM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a824187b143cb8d4b1778d10bcedbce3e',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_58',['IMB_AUTH_AES_CMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aed85279dbff19be59c16216364375a48',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5f256_59',['IMB_AUTH_AES_CMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a0d0cad9513b19110454c9e4e9c549205',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5fbitlen_60',['IMB_AUTH_AES_CMAC_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa9a94b55feed30ce57ecadf17b7c1a83',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_61',['IMB_AUTH_AES_GMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa1496e45f00afe54aeeb527089e14a0d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f128_62',['IMB_AUTH_AES_GMAC_128',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a189af960a1558b7ac3289c0892739ea0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f192_63',['IMB_AUTH_AES_GMAC_192',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adfd8db9df85ef929d9e4a38612f07d66',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f256_64',['IMB_AUTH_AES_GMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae6fab01dbaf8b0110ca8ccdce98547af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fxcbc_65',['IMB_AUTH_AES_XCBC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a17deb61684701b7ec37f68bd34b6d454',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_66',['IMB_AUTH_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab737fa5ff7244ddf5024905f6477ae01',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_5fsgl_67',['IMB_AUTH_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a71119edfbe668d423e910b11c0efb712',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc10_5fiuup_5fdata_68',['IMB_AUTH_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ac27de87b2687b1e2a293544be6d3fab9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc11_5ffp_5fheader_69',['IMB_AUTH_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab0f02ac14a03f137a83bb7d6dff539fc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5ffp_5fdata_70',['IMB_AUTH_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa78e695260f8fe6bb56d242771624ccc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5fx25_71',['IMB_AUTH_CRC16_X25',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a22e2e766cbec475508af01c6a722c2c3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fa_72',['IMB_AUTH_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3a8aa07fe9f70ece22f9351db0f8bb31',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fb_73',['IMB_AUTH_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a77ccdf9abd97310f6808c8cf12c1a395',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fethernet_5ffcs_74',['IMB_AUTH_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a421294f9a35f87e796f96b48257e6dc2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fsctp_75',['IMB_AUTH_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff8b335b5391fc33c80c076590de44',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fwimax_5fofdma_5fdata_76',['IMB_AUTH_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad3936131d6804e6c6aa3bd497ae675ab',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc6_5fiuup_5fheader_77',['IMB_AUTH_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a88175070e077037332ad5cd5fdece46f',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc7_5ffp_5fheader_78',['IMB_AUTH_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab7f1372ade9da040caf195b13f2b79f0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc8_5fwimax_5fofdma_5fhcs_79',['IMB_AUTH_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae62fe411b683840a5f25c199b66c7f42',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcustom_80',['IMB_AUTH_CUSTOM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a70151af8840242c8f5bdb375f2f1135b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fdocsis_5fcrc32_81',['IMB_AUTH_DOCSIS_CRC32',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae006c86697198ca852fc612d376135dc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fgcm_5fsgl_82',['IMB_AUTH_GCM_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ada7d3fc165b6855ae3e83467de9916b2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fghash_83',['IMB_AUTH_GHASH',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a67297eacd4913bd65a1aae650c3a22e3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f1_84',['IMB_AUTH_HMAC_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a441cc344587709b2ff2fc26f36ff1e58',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f224_85',['IMB_AUTH_HMAC_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad0ad42ef3cb8cef38a29bcc667e70fca',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f256_86',['IMB_AUTH_HMAC_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a60c1e71108516903aded4534f0f2e995',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f384_87',['IMB_AUTH_HMAC_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae98f8f8e42b7a026bb2aa2a402a291b9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f512_88',['IMB_AUTH_HMAC_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad8f7bc68ad10fe75665692a27ada55f1',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fkasumi_5fuia1_89',['IMB_AUTH_KASUMI_UIA1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a42dc1dc990059b61ef784813e0f8f49a',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fmd5_90',['IMB_AUTH_MD5',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a958fb278e4745955ebc791ba652954fe',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnull_91',['IMB_AUTH_NULL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a25f47297460775ab891a0abb8240b8e0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnum_92',['IMB_AUTH_NUM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a5d34120f8a1c5cde857baf32536868af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpoly1305_93',['IMB_AUTH_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a93e6c082f7ec5926a087d139a2bd2e56',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpon_5fcrc_5fbip_94',['IMB_AUTH_PON_CRC_BIP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a2fcc080535b4dd16502ecd2b01c14048',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f1_95',['IMB_AUTH_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adcaf4a43c9c114bcdb75ff44ba1bea77',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f224_96',['IMB_AUTH_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3936068075b4572d859c9b1984312325',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f256_97',['IMB_AUTH_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4962dbdf6fcb8d743e4ab55ce23c4b88',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f384_98',['IMB_AUTH_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006afe74859581d19e70b8667773662081bc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f512_99',['IMB_AUTH_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa12aca10078adbe0bc2a021190663e2d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow3g_5fuia2_5fbitlen_100',['IMB_AUTH_SNOW3G_UIA2_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4eadf3270a81eb2567388843da2fa89b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow_5fv_5faead_101',['IMB_AUTH_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff331c62b01fb8427e63e18fc433f6',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc256_5feia3_5fbitlen_102',['IMB_AUTH_ZUC256_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a7c535075abfb0878168a00600a762fe0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc_5feia3_5fbitlen_103',['IMB_AUTH_ZUC_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a188c2297dc5228820e690e8938cd407b',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5ffinalize_104',['IMB_CHACHA20_POLY1305_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a8f6a258b6a13952fad9c217194c716f0',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5fupdate_105',['IMB_CHACHA20_POLY1305_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a4ac8fd429d9f356003522200811b0bed',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5ffinalize_106',['IMB_CHACHA20_POLY1305_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a7575d2fafb2b970bd08a49a556602031',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5fupdate_107',['IMB_CHACHA20_POLY1305_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a2abd7cd460b99467836188d62272cd2b',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5finit_108',['IMB_CHACHA20_POLY1305_INIT',['../intel-ipsec-mb_8h.html#adb8ac3238e6dfe39a10e64602a96e643',1,'intel-ipsec-mb.h']]], + ['imb_5fchain_5forder_109',['IMB_CHAIN_ORDER',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbc_110',['IMB_CIPHER_CBC',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0e2ebfcfd101ab4a39129082b2de517f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbcs_5f1_5f9_111',['IMB_CIPHER_CBCS_1_9',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca71d87659793f7377d1d8b3315a3559f8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fccm_112',['IMB_CIPHER_CCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca7d3946d77e4364bc45a9e72499bf817f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_113',['IMB_CIPHER_CHACHA20',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca39d8d7ebca424a1c1a65bc04b5f64741',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_114',['IMB_CIPHER_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caa5c375f97dce274a2446c8abf90aa61a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_5fsgl_115',['IMB_CIPHER_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca40926d8b731a401ec3e86ffa2089a1eb',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_116',['IMB_CIPHER_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca5b4126e439eaff0975846425594201ea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_5fbitlen_117',['IMB_CIPHER_CNTR_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca6f69815480531651948b5549fa84023a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcustom_118',['IMB_CIPHER_CUSTOM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca4301726c71a4faf412a1a011cc4f0df8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes_119',['IMB_CIPHER_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca049e85047492494fac0a4ec785707851',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes3_120',['IMB_CIPHER_DES3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca639b1382de364a2b810f7876189bc64f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdirection_121',['IMB_CIPHER_DIRECTION',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9ee',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fdes_122',['IMB_CIPHER_DOCSIS_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caf9d62e1ff3a1a8b2ed1dcc515a3ffe1b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fsec_5fbpi_123',['IMB_CIPHER_DOCSIS_SEC_BPI',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caed8ba68d7d2d7602407f17305d1d99b7',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fecb_124',['IMB_CIPHER_ECB',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca357dec895f8dfa3371add1feaa001f56',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_125',['IMB_CIPHER_GCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0f97344e525fef167a92d1a59a0492aa',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_5fsgl_126',['IMB_CIPHER_GCM_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca48e024bc875c46b714b19a1c19908918',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fkasumi_5fuea1_5fbitlen_127',['IMB_CIPHER_KASUMI_UEA1_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3f7a3edb272955efea015c5bb8572e3b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fmode_128',['IMB_CIPHER_MODE',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334c',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnull_129',['IMB_CIPHER_NULL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0bebd24d7311dedce100455dd3a8ddf6',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnum_130',['IMB_CIPHER_NUM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca78c470d405fb36037e41224b99d23c75',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fpon_5faes_5fcntr_131',['IMB_CIPHER_PON_AES_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca896285cd2fe56f73011b908e490cbbea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow3g_5fuea2_5fbitlen_132',['IMB_CIPHER_SNOW3G_UEA2_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca88c3ff9a6816b3629dc5b4815b47607e',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_133',['IMB_CIPHER_SNOW_V',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cafea56410df85a0b006963e50fba78465',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_5faead_134',['IMB_CIPHER_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cada26994f8467300f862dbf45b5a22409',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fzuc_5feea3_135',['IMB_CIPHER_ZUC_EEA3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3618bfaab3c19aabc8a93a03ae121efc',1,'intel-ipsec-mb.h']]], + ['imb_5fclear_5fmem_136',['imb_clear_mem',['../intel-ipsec-mb_8h.html#aecdd69541b6d2a83d19ca804ef4b78d6',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx_137',['IMB_CPUFLAGS_AVX',['../intel-ipsec-mb_8h.html#a0329fede890d95a97423a714410e1576',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_138',['IMB_CPUFLAGS_AVX2',['../intel-ipsec-mb_8h.html#ac0a92622a85423541536321231a6cb7d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_5ft2_139',['IMB_CPUFLAGS_AVX2_T2',['../intel-ipsec-mb_8h.html#a18f17a6bcef83c3976ca7a9b4ca624de',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_5ft3_140',['IMB_CPUFLAGS_AVX2_T3',['../intel-ipsec-mb_8h.html#a6e356586d2d3d7e93ff1b23d4a90b344',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx512_141',['IMB_CPUFLAGS_AVX512',['../intel-ipsec-mb_8h.html#a0a250932ede6ca7b69f5c763c24cf51d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx512_5ft2_142',['IMB_CPUFLAGS_AVX512_T2',['../intel-ipsec-mb_8h.html#af7bdbff703949eb2cf2dd3d3a08803de',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx_5ft2_143',['IMB_CPUFLAGS_AVX_T2',['../intel-ipsec-mb_8h.html#a85dab1abb0996e9ff4d4258fef3f9abd',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fno_5faesni_144',['IMB_CPUFLAGS_NO_AESNI',['../intel-ipsec-mb_8h.html#a6afb888ab232bab1c3ca027eb071f388',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_145',['IMB_CPUFLAGS_SSE',['../intel-ipsec-mb_8h.html#a5654f563aaad731df22c8e8517c9154c',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_5ft2_146',['IMB_CPUFLAGS_SSE_T2',['../intel-ipsec-mb_8h.html#ae6776750b801d7e177c152569c10c546',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_5ft3_147',['IMB_CPUFLAGS_SSE_T3',['../intel-ipsec-mb_8h.html#aa1cc7ac79ca44f06e3caa18fdd6d441a',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc10_5fiuup_5fdata_148',['IMB_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#ac7406ab5572b4af93ff8dbbc8ee0c961',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc11_5ffp_5fheader_149',['IMB_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#ae35b6b1b19ef6d599d47591f6c1e1732',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5ffp_5fdata_150',['IMB_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#abe747d2365eb0393ab748c218029e3c6',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5fx25_151',['IMB_CRC16_X25',['../intel-ipsec-mb_8h.html#a78217e8116876c0ecbe82fd3d417cc1c',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fa_152',['IMB_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a6ff60117e0ff260865cd80323306479b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fb_153',['IMB_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#ae07e15be0652ca8faa23c6f907d83e03',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fethernet_5ffcs_154',['IMB_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a075757c7f7b132a8f2ae13c18f36e2af',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fsctp_155',['IMB_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a86753ed15fcd8aab177d276dd4ebf169',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fwimax_5fofdma_5fdata_156',['IMB_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#ab0a9f19127f3417cc78cfdf9091af33e',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc6_5fiuup_5fheader_157',['IMB_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#ac7c45b0788456473390ca178275f9a34',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc7_5ffp_5fheader_158',['IMB_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a23fe73b1f9d40348bc98f92f859df60b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc8_5fwimax_5fofdma_5fhcs_159',['IMB_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a0edd5fa8f3351ee214f7f2a267949f6d',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fblock_5fsize_160',['IMB_DES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a95f4ece0b318cb36bdbe66089d9ef649',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkey_5fsched_5fsize_161',['IMB_DES_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a7d156b1b9d715299e08cca76e790e2b7',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkeysched_162',['IMB_DES_KEYSCHED',['../intel-ipsec-mb_8h.html#a6fbb59f06f60f1900b591735ad691ac6',1,'intel-ipsec-mb.h']]], + ['imb_5fdim_163',['IMB_DIM',['../intel-ipsec-mb_8h.html#a1beec59f9694a41c22b6b099263c070d',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fdecrypt_164',['IMB_DIR_DECRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eea881652011955c227d94a881d2d2c5c59',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fencrypt_165',['IMB_DIR_ENCRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eeabed12e936823e8f8b2de70388a2d15ce',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5fexport_166',['IMB_DLL_EXPORT',['../intel-ipsec-mb_8h.html#a986643689f56109161597f67eb9d7b40',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5flocal_167',['IMB_DLL_LOCAL',['../intel-ipsec-mb_8h.html#a479e22216a6be5645af96c4b1433339e',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5fmin_5feth_5fpdu_5fsize_168',['IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE',['../intel-ipsec-mb_8h.html#a74dca2bb8b0c87dec90f3b985c0266af',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5ftag_5fsize_169',['IMB_DOCSIS_CRC32_TAG_SIZE',['../intel-ipsec-mb_8h.html#a59c620471ea8c9577ed593127f82c758',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_170',['IMB_ERR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0b',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5faad_5flen_171',['IMB_ERR_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0ad3ade3fe22c5887d9bf444ff3923aa',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5flen_172',['IMB_ERR_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafe5af1c17e1caece14b3a5f659a1336d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5ftag_5flen_173',['IMB_ERR_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6af5bd8fb7690b46afe3d2c1e623808e',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fburst_5fooo_174',['IMB_ERR_BURST_OOO',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae4584575cf7134ba02ca1d38bb3f6444',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fburst_5fsize_175',['IMB_ERR_BURST_SIZE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae2bc238b157942636836eaf0eedb5ddf',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fburst_5fsuite_5fid_176',['IMB_ERR_BURST_SUITE_ID',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba436eb6f9e045a02e3ad5998da8c13d78',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5flen_177',['IMB_ERR_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bab5a2b4c1df7778c3efc236976062ed94',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5fmode_178',['IMB_ERR_CIPH_MODE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba61a411feda0e8cf5aff8c58ad2e92bdd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fhash_5falgo_179',['IMB_ERR_HASH_ALGO',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8ac2802661bd9928d0c766129db2a814',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fiv_5flen_180',['IMB_ERR_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baac8500b58709f0bd05ad129337bb6af2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5faad_5flen_181',['IMB_ERR_JOB_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba31a6b672b95d7d100a97fa6d788c0dd6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5flen_182',['IMB_ERR_JOB_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69dd17473dc4afb2f04c417fe6882608',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5ftag_5flen_183',['IMB_ERR_JOB_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1b7e4c43808275e9cd63ca21091e5ab2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fchain_5forder_184',['IMB_ERR_JOB_CHAIN_ORDER',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacaec18a061b4bf792178408d69775950',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5fdir_185',['IMB_ERR_JOB_CIPH_DIR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacbf5d148be9963df1d46400ff26396db',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5flen_186',['IMB_ERR_JOB_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae1c9e58d4be3fdd7bb37247667472920',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fiv_5flen_187',['IMB_ERR_JOB_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba390f91827f87bbd3233ad6e1f104b2b4',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fkey_5flen_188',['IMB_ERR_JOB_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badfea107da94936d30900f23ad22b5285',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5faad_189',['IMB_ERR_JOB_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3ef3d3180e3052862c796cf89320d60c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_190',['IMB_ERR_JOB_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba503544043c60c2195003c00b18141beb',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_5fkey_191',['IMB_ERR_JOB_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba97fb89f93299f67de86aac919bb83aa9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fdst_192',['IMB_ERR_JOB_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba55b3ddc1a48cb14ad020031b8d5b4927',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fghash_5finit_5ftag_193',['IMB_ERR_JOB_NULL_GHASH_INIT_TAG',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6c97704719f286b14204ca144bbdc7a9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fipad_194',['IMB_ERR_JOB_NULL_HMAC_IPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba15f55e87bb42cf5494dd01b9a1c4d712',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fopad_195',['IMB_ERR_JOB_NULL_HMAC_OPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacb9286af7533993ab79982b288aab97a',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fiv_196',['IMB_ERR_JOB_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0d35070d6476cde0202deb5531ae09c2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fkey_197',['IMB_ERR_JOB_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba22402460340d6b34492ece390a7913c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fnext_5fiv_198',['IMB_ERR_JOB_NULL_NEXT_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba983561202b1623b0bd1f691bd6036039',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsgl_5fctx_199',['IMB_ERR_JOB_NULL_SGL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba05f9212af539a22ad65b12a9abc8e5c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsrc_200',['IMB_ERR_JOB_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4bc6a784afa9bb88c5afe4fd1b5f610d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk1_5fexp_201',['IMB_ERR_JOB_NULL_XCBC_K1_EXP',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69bf9e54fdf84b154284eab071c98804',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk2_202',['IMB_ERR_JOB_NULL_XCBC_K2',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bad7fbd09092821fb8605e11384dcdc428',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk3_203',['IMB_ERR_JOB_NULL_XCBC_K3',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baf12955208a99c8feeeb580fee940b7c3',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fpon_5fpli_204',['IMB_ERR_JOB_PON_PLI',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bace7e834842ddef5b063c99544d00b303',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fsrc_5foffset_205',['IMB_ERR_JOB_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba207979780b5d8472dfef2f5e12998d22',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fkey_5flen_206',['IMB_ERR_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8d6b3ea721216bb12fbe40e72a5bd768',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmax_207',['IMB_ERR_MAX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bac773be615f9904fd69e961451fb0fac6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmin_208',['IMB_ERR_MIN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3f9d55f356a0caf413d2114a468ba659',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmissing_5fcpuflags_5finit_5fmgr_209',['IMB_ERR_MISSING_CPUFLAGS_INIT_MGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0babdd0b7bc92e7dcf166d36a9bcd118d60',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fno_5faesni_5femu_210',['IMB_ERR_NO_AESNI_EMU',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4496bf14e2ddf1133a384f9fbf96d5dc',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5faad_211',['IMB_ERR_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badc7d17d90012d5c14fd8a12d9ea12543',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_212',['IMB_ERR_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1e20954ec8a501093175d640631550c1',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_5fkey_213',['IMB_ERR_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baa4f5aeab78ed40a4c19389b8f691306c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fburst_214',['IMB_ERR_NULL_BURST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba99d845a176cd571dd6f99b0a430291c6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fctx_215',['IMB_ERR_NULL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0b57ec134f6e10e11c3fbcf77ede282d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fdst_216',['IMB_ERR_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1a61dcc99a1a9725c89076dec3c02e02',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fexp_5fkey_217',['IMB_ERR_NULL_EXP_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3b60a570372529486da69c6d2b7be382',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fiv_218',['IMB_ERR_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafcfe882ee9730fd1724b587106f2f419',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fjob_219',['IMB_ERR_NULL_JOB',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba007c436f8e21f317e8f9225c5b009d38',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fkey_220',['IMB_ERR_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba63029e3673eef74345ad80867f7ba0ef',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fmbmgr_221',['IMB_ERR_NULL_MBMGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba9ba69cc32a2d17e3e7b4d4c982f26abd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fsrc_222',['IMB_ERR_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba308819b7cbdadfd6bbee290408880302',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fqueue_5fspace_223',['IMB_ERR_QUEUE_SPACE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4f73ed8232981b84df1fee21024e01ac',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fselftest_224',['IMB_ERR_SELFTEST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1873d38b8aaf5ebcd3f542b15c418829',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fsrc_5foffset_225',['IMB_ERR_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba20dfbdc8feb8332a9efe51de48fa6e09',1,'intel-ipsec-mb.h']]], + ['imb_5ferrno_226',['imb_errno',['../structIMB__MGR.html#a0ab7a3dfdfd8442f591699e2310df11f',1,'IMB_MGR']]], + ['imb_5ffeature_5faesni_227',['IMB_FEATURE_AESNI',['../intel-ipsec-mb_8h.html#a19c147087d2ae918baa22f90fa9b3b05',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5faesni_5femu_228',['IMB_FEATURE_AESNI_EMU',['../intel-ipsec-mb_8h.html#a421158db6d19c97a6bceb32eb4a839ed',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx_229',['IMB_FEATURE_AVX',['../intel-ipsec-mb_8h.html#a7af992d12b4f9a6e789384969a066056',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx2_230',['IMB_FEATURE_AVX2',['../intel-ipsec-mb_8h.html#a225e2cd9c1445795fc62879eabad2530',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fifma_231',['IMB_FEATURE_AVX512_IFMA',['../intel-ipsec-mb_8h.html#a0134dfaffe12427a8f2182247c6f1f48',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fskx_232',['IMB_FEATURE_AVX512_SKX',['../intel-ipsec-mb_8h.html#ae6d6e3427c81b278d64f0189401c4964',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512bw_233',['IMB_FEATURE_AVX512BW',['../intel-ipsec-mb_8h.html#a00caae81d425acb6b946bebb25ac267f',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512cd_234',['IMB_FEATURE_AVX512CD',['../intel-ipsec-mb_8h.html#abbdb390b80ff1a92f31a7a9adc9beeb9',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512dq_235',['IMB_FEATURE_AVX512DQ',['../intel-ipsec-mb_8h.html#a8ca2c970faa0f62a3f4a6b6c18432bb1',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512f_236',['IMB_FEATURE_AVX512F',['../intel-ipsec-mb_8h.html#ab673c08624d868ca0dd89e1a9ec28f00',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512vl_237',['IMB_FEATURE_AVX512VL',['../intel-ipsec-mb_8h.html#a803871900c08e2dc1b3f805ed4ecefd8',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx_5fifma_238',['IMB_FEATURE_AVX_IFMA',['../intel-ipsec-mb_8h.html#a8a946f162ff5e363d04f8b44d2f48747',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fbmi2_239',['IMB_FEATURE_BMI2',['../intel-ipsec-mb_8h.html#a557f3de4bcc0bd7e9bd90e5e1d1b9b09',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fcmov_240',['IMB_FEATURE_CMOV',['../intel-ipsec-mb_8h.html#a5eef394935c651931b1cf560f31c841c',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fgfni_241',['IMB_FEATURE_GFNI',['../intel-ipsec-mb_8h.html#aa67ab4088ac2f8cadbe839101d276567',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fpclmulqdq_242',['IMB_FEATURE_PCLMULQDQ',['../intel-ipsec-mb_8h.html#a1af54a4a862936a6793af2ff44163ba5',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fdata_243',['IMB_FEATURE_SAFE_DATA',['../intel-ipsec-mb_8h.html#abd58fdbe1f1ef53cf509a0b37ba8aa4b',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fparam_244',['IMB_FEATURE_SAFE_PARAM',['../intel-ipsec-mb_8h.html#a4e9506dc72ab4e6a2af463238ccb3080',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fself_5ftest_245',['IMB_FEATURE_SELF_TEST',['../intel-ipsec-mb_8h.html#a8b4181547ded3ff17859d2632c468151',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fself_5ftest_5fpass_246',['IMB_FEATURE_SELF_TEST_PASS',['../intel-ipsec-mb_8h.html#a673b969c5f01579f6ca0a6b003cf1197',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fshani_247',['IMB_FEATURE_SHANI',['../intel-ipsec-mb_8h.html#acb9c15eabfb9bafeb15a06886fa91e90',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsse4_5f2_248',['IMB_FEATURE_SSE4_2',['../intel-ipsec-mb_8h.html#abcc243fc505aca3eac6963eff25e9b99',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvaes_249',['IMB_FEATURE_VAES',['../intel-ipsec-mb_8h.html#af4321e4e3c947f9b9fb7f204aaecf2c3',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvpclmulqdq_250',['IMB_FEATURE_VPCLMULQDQ',['../intel-ipsec-mb_8h.html#a153d695ecd1af69a0d23385a50654614',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5faesni_5foff_251',['IMB_FLAG_AESNI_OFF',['../intel-ipsec-mb_8h.html#a4c53fb8e39bbc6e3f39d0558934e5d55',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5fgfni_5foff_252',['IMB_FLAG_GFNI_OFF',['../intel-ipsec-mb_8h.html#ad3faac9d03a87621ae02c10337b4edf8',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5fshani_5foff_253',['IMB_FLAG_SHANI_OFF',['../intel-ipsec-mb_8h.html#a3d45fcff99a9ea952b4dd6ef70e36e0c',1,'intel-ipsec-mb.h']]], + ['imb_5fflush_5fburst_254',['IMB_FLUSH_BURST',['../intel-ipsec-mb_8h.html#afff52a9aaa1c1875e7c905e2fb29fc43',1,'intel-ipsec-mb.h']]], + ['imb_5fflush_5fjob_255',['IMB_FLUSH_JOB',['../intel-ipsec-mb_8h.html#a4726e3683b3579b1e643134278386d72',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f128_5fkey_5flen_256',['IMB_GCM_128_KEY_LEN',['../intel-ipsec-mb_8h.html#ae97681c203afbd818a39fb797613a232',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f192_5fkey_5flen_257',['IMB_GCM_192_KEY_LEN',['../intel-ipsec-mb_8h.html#a7201a0be67f1e6f3757169b0166dd47d',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f256_5fkey_5flen_258',['IMB_GCM_256_KEY_LEN',['../intel-ipsec-mb_8h.html#a4c4c40ec87c2354832f5b182b22d2111',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fblock_5flen_259',['IMB_GCM_BLOCK_LEN',['../intel-ipsec-mb_8h.html#a59def6ed420a4ef91ce4a6562cc2c4d5',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fenc_5fkey_5flen_260',['IMB_GCM_ENC_KEY_LEN',['../intel-ipsec-mb_8h.html#a0c6f54ab6663187fa401c766a981a41e',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fiv_5fdata_5flen_261',['IMB_GCM_IV_DATA_LEN',['../intel-ipsec-mb_8h.html#a87884da318b3013893b725d047b53702',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fkey_5fsets_262',['IMB_GCM_KEY_SETS',['../intel-ipsec-mb_8h.html#a73715b8c1ba918ce6d5c21749841b47b',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fcompleted_5fjob_263',['IMB_GET_COMPLETED_JOB',['../intel-ipsec-mb_8h.html#a18e5a7f85573f0006448deda41014117',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ferrno_264',['imb_get_errno',['../intel-ipsec-mb_8h.html#a79f60b2bffbe91dac3e4f8b486e7ab10',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ffeature_5fflags_265',['imb_get_feature_flags',['../intel-ipsec-mb_8h.html#a725703ff6da6b402bde7041a2fabfddf',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fmb_5fmgr_5fsize_266',['imb_get_mb_mgr_size',['../intel-ipsec-mb_8h.html#ac8379ada0d6ec030ebad9828fbea9459',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fnext_5fburst_267',['IMB_GET_NEXT_BURST',['../intel-ipsec-mb_8h.html#a6029af3f88d1908dd11a8f3e9eb9fd70',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fnext_5fjob_268',['IMB_GET_NEXT_JOB',['../intel-ipsec-mb_8h.html#a34b6006fc39dd4ea23d3fec49237d382',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fstrerror_269',['imb_get_strerror',['../intel-ipsec-mb_8h.html#a61f664448a862fbb02e728ce518ff8e4',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_270',['imb_get_version',['../intel-ipsec-mb_8h.html#aee02bcb3a5e5eaee3632086f75c3d78d',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_5fstr_271',['imb_get_version_str',['../intel-ipsec-mb_8h.html#a4490e7b324236315f1a9e141b39f99f2',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_272',['IMB_GHASH',['../intel-ipsec-mb_8h.html#a477ad08a4c55e2c6458b7e86dae803ec',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_5fpre_273',['IMB_GHASH_PRE',['../intel-ipsec-mb_8h.html#a230f16784519d7e2b69de87871d35cb2',1,'intel-ipsec-mb.h']]], + ['imb_5fhash_5falg_274',['IMB_HASH_ALG',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f32_275',['IMB_HEC_32',['../intel-ipsec-mb_8h.html#a37996cede66ee8036b1b113af5f553ac',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f64_276',['IMB_HEC_64',['../intel-ipsec-mb_8h.html#aa0b62ba2c9fb07a1ad3c8105ec4374a4',1,'intel-ipsec-mb.h']]], + ['imb_5fhmac_5fipad_5fopad_277',['imb_hmac_ipad_opad',['../intel-ipsec-mb_8h.html#ab5ecb9fd270d0bfc42e46c5a3a9225fe',1,'intel-ipsec-mb.h']]], + ['imb_5fjob_278',['IMB_JOB',['../structIMB__JOB.html',1,'IMB_JOB'],['../intel-ipsec-mb_8h.html#aed2387ba36d72c3d148fcb9d2802ccf5',1,'IMB_JOB: intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fblock_5fsize_279',['IMB_KASUMI_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#aa31ca303e03fc548199f95e5335889e1',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fdigest_5fsize_280',['IMB_KASUMI_DIGEST_SIZE',['../intel-ipsec-mb_8h.html#a56bd44d01c91469b57efa36f360a83bf',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_281',['IMB_KASUMI_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a45ac331e1f9241e53fc35a1f88dec6ce',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_5fbit_282',['IMB_KASUMI_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#ae3befb703f7dec2c0695e61542f3212c',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f2_5fbuffer_283',['IMB_KASUMI_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a884bfa72fc232c7e84043d0faee719ee',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f3_5fbuffer_284',['IMB_KASUMI_F8_3_BUFFER',['../intel-ipsec-mb_8h.html#af35e87948218048e4bbad7162208fe41',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f4_5fbuffer_285',['IMB_KASUMI_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a7ed6be7362e5ff7553fb4e7633bee76a',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5fn_5fbuffer_286',['IMB_KASUMI_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a65b1afbadee23111046746a4d302bb0d',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_287',['IMB_KASUMI_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#a43c13bbdf83e2c75d6dd29d01dec4a1f',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_5fuser_288',['IMB_KASUMI_F9_1_BUFFER_USER',['../intel-ipsec-mb_8h.html#a66dcf1985c046ef619141a6706159940',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff8_5fkey_5fsched_289',['IMB_KASUMI_INIT_F8_KEY_SCHED',['../intel-ipsec-mb_8h.html#aafcc540a047a6036adf4da0613ea5272',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff9_5fkey_5fsched_290',['IMB_KASUMI_INIT_F9_KEY_SCHED',['../intel-ipsec-mb_8h.html#a80a71ab95fd99c8427e171e45cebbd92',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fiv_5fsize_291',['IMB_KASUMI_IV_SIZE',['../intel-ipsec-mb_8h.html#adf7e13e43c6bb30489d0e26f11e39440',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsched_5fsize_292',['IMB_KASUMI_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#ae3812840fbe6b9b5ea5e84a6b86a7ed3',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsize_293',['IMB_KASUMI_KEY_SIZE',['../intel-ipsec-mb_8h.html#a46d59512e3a7254e54018181b7a31102',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f128_5fbytes_294',['IMB_KEY_128_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaf3e82db9a8b5b1b229c201735938969d',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f192_5fbytes_295',['IMB_KEY_192_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cca573d62aa6a04d9ce2b3c34e226846598',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f256_5fbytes_296',['IMB_KEY_256_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccac139e7555c9b30725359698833eb213a',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f64_5fbytes_297',['IMB_KEY_64_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaa1eb9eb83fca5f5dd6578f27733eb390',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5fsize_5fbytes_298',['IMB_KEY_SIZE_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cc',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5fburst_5fsize_299',['IMB_MAX_BURST_SIZE',['../intel-ipsec-mb_8h.html#af9070daf9749490016e9b5106b338867',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5fjobs_300',['IMB_MAX_JOBS',['../intel-ipsec-mb_8h.html#ab12d22470a68b5d3852c8dd972784d97',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5ftag_5flen_301',['IMB_MAX_TAG_LEN',['../intel-ipsec-mb_8h.html#a41087ea6dce2e096ebb02a613600c2ed',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fblock_5fsize_302',['IMB_MD5_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a567100ef1c32f08b9e75e1fb378171a3',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fdigest_5fsize_5fin_5fbytes_303',['IMB_MD5_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a472528903f010fe2c365d1ff23565eaa',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fone_5fblock_304',['IMB_MD5_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a52b27293e2a19809c5f7d0f2afab97a2',1,'intel-ipsec-mb.h']]], + ['imb_5fmgr_305',['IMB_MGR',['../structIMB__MGR.html',1,'IMB_MGR'],['../intel-ipsec-mb_8h.html#a6d466496b4adfea3f9bc0881f11fe551',1,'IMB_MGR: intel-ipsec-mb.h']]], + ['imb_5forder_5fcipher_5fhash_306',['IMB_ORDER_CIPHER_HASH',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5e3ab845752ce3d16e40ea7802ce4a52',1,'intel-ipsec-mb.h']]], + ['imb_5forder_5fhash_5fcipher_307',['IMB_ORDER_HASH_CIPHER',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5de8d032616267941d99dad14801044d',1,'intel-ipsec-mb.h']]], + ['imb_5fqueue_5fsize_308',['IMB_QUEUE_SIZE',['../intel-ipsec-mb_8h.html#a712f6e7ecb2a16f8e1bc96b84d188365',1,'intel-ipsec-mb.h']]], + ['imb_5fquic_5faes_5fgcm_309',['imb_quic_aes_gcm',['../intel-ipsec-mb_8h.html#a8faae8f9387983904125d862c4cac9ab',1,'intel-ipsec-mb.h']]], + ['imb_5fquic_5fhp_5faes_5fecb_310',['imb_quic_hp_aes_ecb',['../intel-ipsec-mb_8h.html#a89f9807d10f3459444bb649caf9b64ca',1,'intel-ipsec-mb.h']]], + ['imb_5fset_5fpointers_5fmb_5fmgr_311',['imb_set_pointers_mb_mgr',['../intel-ipsec-mb_8h.html#a3a825e903d1f8f8611b98b60fcaa5a81',1,'intel-ipsec-mb.h']]], + ['imb_5fset_5fsession_312',['imb_set_session',['../intel-ipsec-mb_8h.html#ada304337e78ef5a462c97c1f8151ef67',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fall_313',['IMB_SGL_ALL',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da2f0678ab7ad9bb30c040e0b3ee2fa408',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fcomplete_314',['IMB_SGL_COMPLETE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3dab7a602903c0fbaad7ffb893cb2547975',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5finit_315',['IMB_SGL_INIT',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3daf6e77a48759ab366cc1160725ff4261e',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fiov_316',['IMB_SGL_IOV',['../structIMB__SGL__IOV.html',1,'']]], + ['imb_5fsgl_5fstate_317',['IMB_SGL_STATE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3d',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fupdate_318',['IMB_SGL_UPDATE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da15c5d0aeeb9d5eb4e4c4878c837c5977',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_319',['IMB_SHA1',['../intel-ipsec-mb_8h.html#ae61900d14cde2836621a1421141548a4',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fblock_5fsize_320',['IMB_SHA1_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#af19fd9d6768e33720cab4ecd9aabad91',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fdigest_5fsize_5fin_5fbytes_321',['IMB_SHA1_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a8c821d8cf5520f6f46e9b835eeeb2066',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fone_5fblock_322',['IMB_SHA1_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ac2f9695b12c32c303faa7c93c1c24c9b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_323',['IMB_SHA224',['../intel-ipsec-mb_8h.html#a542a40ced46bf97a678c6b44c325b360',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fdigest_5fsize_5fin_5fbytes_324',['IMB_SHA224_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a836641341e86b80f67716eabdf4a64c5',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fone_5fblock_325',['IMB_SHA224_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ab1f7eb5e9bcd0c69a70dc76747f919f6',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_326',['IMB_SHA256',['../intel-ipsec-mb_8h.html#a37338b9f91315e5de0c4f3d19859aa7b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fdigest_5fsize_5fin_5fbytes_327',['IMB_SHA256_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a4bc9eee15ba0ce09f545af8916292731',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fone_5fblock_328',['IMB_SHA256_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a6023b87b5d1c6dc7dc0547db540af997',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_329',['IMB_SHA384',['../intel-ipsec-mb_8h.html#ac8052b8b996f866f27b4d443bcda0d5f',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fdigest_5fsize_5fin_5fbytes_330',['IMB_SHA384_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a2538a5e82141eb5a54f78660241e186c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fone_5fblock_331',['IMB_SHA384_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a763d0cfb16263906e44f46696b16124b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_332',['IMB_SHA512',['../intel-ipsec-mb_8h.html#ac806d6b629633e801409d3e363d83f6c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fdigest_5fsize_5fin_5fbytes_333',['IMB_SHA512_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a9efcf370f0db0b21f7cf4a3b72eec53e',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fone_5fblock_334',['IMB_SHA512_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a71d0a529b355a434efeeda2e01870ba9',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f224_5fblock_5fsize_335',['IMB_SHA_224_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a7c912b7543122de6f05355fa4848eb83',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f256_5fblock_5fsize_336',['IMB_SHA_256_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a7eed0159b050f8d78cddcc78b4b941ef',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f384_5fblock_5fsize_337',['IMB_SHA_384_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a256cc0f9eaba38030ca3f7589e479855',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f512_5fblock_5fsize_338',['IMB_SHA_512_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a4141433b17ade98e14a585e0409cb62d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_339',['IMB_SNOW3G_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a4bcfbb70da38433f8bcd3468779026f1',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_5fbit_340',['IMB_SNOW3G_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#a5d7a9e694b932b1c198782e22dc1ead5',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f2_5fbuffer_341',['IMB_SNOW3G_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a51085837f0775feae26fe3885f7459b7',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f4_5fbuffer_342',['IMB_SNOW3G_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a79b873457a74ad6c50e689709d79a0fb',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_343',['IMB_SNOW3G_F8_8_BUFFER',['../intel-ipsec-mb_8h.html#a8536e68d053999123eb104762ef51932',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_5fmultikey_344',['IMB_SNOW3G_F8_8_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#aca8fb428a3b291950cd9dc6277ad801a',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_345',['IMB_SNOW3G_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a4538b34eaf3a3d402041ada1e8d03e2c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_5fmultikey_346',['IMB_SNOW3G_F8_N_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#a4a5ae1b04c7aee099c2f8fcbd488c38c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff9_5f1_5fbuffer_347',['IMB_SNOW3G_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#aa6a951033156fc06c25afb4f79de8851',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5finit_5fkey_5fsched_348',['IMB_SNOW3G_INIT_KEY_SCHED',['../intel-ipsec-mb_8h.html#ac5b5ff1cc9cf44ea8587a7d054b7459d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5fkey_5fsched_5fsize_349',['IMB_SNOW3G_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a1ddd766994cb5d0b8e9ef5cd580cdcd2',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_350',['IMB_STATUS',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066ab',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fbeing_5fprocessed_351',['IMB_STATUS_BEING_PROCESSED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaa562b01ade81e0196fc23cd79d534141',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_352',['IMB_STATUS_COMPLETED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba508a9f195acee09621542ca687ba2341',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fauth_353',['IMB_STATUS_COMPLETED_AUTH',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba79df83a15cdafc2a6c79751457ae9792',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fcipher_354',['IMB_STATUS_COMPLETED_CIPHER',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba6fcaccd3853c9e8cf3b402edc3e22089',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5ferror_355',['IMB_STATUS_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba0ef40895610ac6343d171f65be676a1d',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finternal_5ferror_356',['IMB_STATUS_INTERNAL_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba546465b9d4cfb1f9c98c864bf5ed51fb',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finvalid_5fargs_357',['IMB_STATUS_INVALID_ARGS',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaee85b6a8bc33e0b39001e9ff538540b7',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_358',['IMB_SUBMIT_BURST',['../intel-ipsec-mb_8h.html#aabfad78b032b6b160e3ba51477132524',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_5fnocheck_359',['IMB_SUBMIT_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#aa06f0e18794d4a95d16e61dfb1af70c4',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_360',['IMB_SUBMIT_CIPHER_BURST',['../intel-ipsec-mb_8h.html#ae487720f5112a3ecec659ab4aeada77e',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_5fnocheck_361',['IMB_SUBMIT_CIPHER_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#af819e81f584b3b56e0201a7174b3c0eb',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_362',['IMB_SUBMIT_HASH_BURST',['../intel-ipsec-mb_8h.html#a3702b37004e38dd27a9b8a008a0c48ed',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_5fnocheck_363',['IMB_SUBMIT_HASH_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#afd6b80655a6129fe063d2ffb391e3015',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_364',['IMB_SUBMIT_JOB',['../intel-ipsec-mb_8h.html#a3602a4ccaf1da4bf6787169370cef04f',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_5fnocheck_365',['IMB_SUBMIT_JOB_NOCHECK',['../intel-ipsec-mb_8h.html#af23c64e466272c5ae90fcf78fbcd0c3d',1,'intel-ipsec-mb.h']]], + ['imb_5fuint128_5ft_366',['imb_uint128_t',['../structimb__uint128__t.html',1,'']]], + ['imb_5fversion_367',['IMB_VERSION',['../intel-ipsec-mb_8h.html#ad9bcb21181d1e07b4f66d8bf0514c1a3',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fnum_368',['IMB_VERSION_NUM',['../intel-ipsec-mb_8h.html#a415abfc6067c0b1663a508993a4eb0d4',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fstr_369',['IMB_VERSION_STR',['../intel-ipsec-mb_8h.html#a874972638400dafdb542319d61bfbfbc',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fdigest_5flen_5fin_5fbytes_5fmax_370',['IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX',['../intel-ipsec-mb_8h.html#a2b1faea7079fa3ddb2cabeb41aeb2a0a',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fdigest_5flen_5fin_5fbytes_5fmin_371',['IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN',['../intel-ipsec-mb_8h.html#a3dc24506ef16ad090414608c762b3d3c',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fiv_5flen_5fin_5fbytes_5fmax_372',['IMB_ZUC256_IV_LEN_IN_BYTES_MAX',['../intel-ipsec-mb_8h.html#a5ab93cc1831b961f3d04ff913f63b832',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fiv_5flen_5fin_5fbytes_5fmin_373',['IMB_ZUC256_IV_LEN_IN_BYTES_MIN',['../intel-ipsec-mb_8h.html#a18e1733291bb917240ef117f52c6aa6c',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fkey_5flen_5fin_5fbytes_374',['IMB_ZUC256_KEY_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#a35d04f4dc107531e37fcab1fd5efbf93',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5fdigest_5flen_5fin_5fbytes_375',['IMB_ZUC_DIGEST_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#adbb9f3bfdc1d416c20a579eb3a8b0b8f',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f1_5fbuffer_376',['IMB_ZUC_EEA3_1_BUFFER',['../intel-ipsec-mb_8h.html#adb23f957dde69fb3bb4c6798d1a69da4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f4_5fbuffer_377',['IMB_ZUC_EEA3_4_BUFFER',['../intel-ipsec-mb_8h.html#a57445dd863838d04cf44f4d79c4dbcaf',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5fn_5fbuffer_378',['IMB_ZUC_EEA3_N_BUFFER',['../intel-ipsec-mb_8h.html#a6382e3185f31e61f995ab90110c52991',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5f1_5fbuffer_379',['IMB_ZUC_EIA3_1_BUFFER',['../intel-ipsec-mb_8h.html#ac789fe54f3a8f92144831a67b14501d4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5fn_5fbuffer_380',['IMB_ZUC_EIA3_N_BUFFER',['../intel-ipsec-mb_8h.html#ae840d62f1246c0c4270c3319ef356d3c',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5fiv_5flen_5fin_5fbytes_381',['IMB_ZUC_IV_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#aaacb0749fd75d1133414685b5694652d',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5fkey_5flen_5fin_5fbytes_382',['IMB_ZUC_KEY_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#a053b0646f055d7ee3be974bb2bfe534b',1,'intel-ipsec-mb.h']]], + ['in_383',['in',['../structIMB__SGL__IOV.html#a56c5b0a033d949869cee2d03f1df3129',1,'IMB_SGL_IOV']]], + ['in_5flength_384',['in_length',['../structgcm__context__data.html#a0d9e72f85aba9363fcb4f2d757342aa2',1,'gcm_context_data']]], + ['init_5fmb_5fmgr_5fauto_385',['init_mb_mgr_auto',['../intel-ipsec-mb_8h.html#a2cbbd032835837fd0aa83b04f5337638',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx_386',['init_mb_mgr_avx',['../intel-ipsec-mb_8h.html#aee547b1b9e70e12c84d9949317e41c6c',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx2_387',['init_mb_mgr_avx2',['../intel-ipsec-mb_8h.html#a71b6563b6b4c0c504cd9b8302a242cfa',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx512_388',['init_mb_mgr_avx512',['../intel-ipsec-mb_8h.html#a98932a3c61d798b0c8a560dcc3dce16f',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5fsse_389',['init_mb_mgr_sse',['../intel-ipsec-mb_8h.html#a2ad1941fe5d77b8bb2d6445679e3b0c5',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5ft_390',['init_mb_mgr_t',['../intel-ipsec-mb_8h.html#af0c38ca0a5aac81d8db7e0cc2811426b',1,'intel-ipsec-mb.h']]], + ['intel_2dipsec_2dmb_391',['intel-ipsec-mb',['../index.html',1,'']]], + ['intel_2dipsec_2dmb_2eh_392',['intel-ipsec-mb.h',['../intel-ipsec-mb_8h.html',1,'']]], + ['iv_393',['iv',['../structIMB__JOB.html#a960fdd62c1c10d2d3cc0a765c0a6531d',1,'IMB_JOB']]], + ['iv_394',['IV',['../structchacha20__poly1305__context__data.html#ac0d1657fc57308900691369362c56658',1,'chacha20_poly1305_context_data']]], + ['iv_5flen_5fin_5fbytes_395',['iv_len_in_bytes',['../structIMB__JOB.html#a25b4ce8aec20f64cfc393b245d62a09e',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_a.js b/docs/search/all_a.js new file mode 100644 index 0000000000000000000000000000000000000000..f0cb16fefd6279e273ced34f7a66eaf7c1a372d2 --- /dev/null +++ b/docs/search/all_a.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['jobs_0',['jobs',['../structIMB__MGR.html#ab342224dce812fb13a9e935e7a7b0617',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_b.js b/docs/search/all_b.js new file mode 100644 index 0000000000000000000000000000000000000000..4e6bfd2ee71406394d2ba07041dd70f1eafed3ba --- /dev/null +++ b/docs/search/all_b.js @@ -0,0 +1,29 @@ +var searchData= +[ + ['k_0',['k',['../structsnow3g__key__schedule__s.html#ae0ca8cd2581836c0297b34eea1effb12',1,'snow3g_key_schedule_s']]], + ['kasumi_5ff8_5f1_5fbuffer_5fbit_5ft_1',['kasumi_f8_1_buffer_bit_t',['../intel-ipsec-mb_8h.html#a81afa744fb5669fe37cf0dce2193d8a7',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f1_5fbuffer_5ft_2',['kasumi_f8_1_buffer_t',['../intel-ipsec-mb_8h.html#a89fe975f0388b71765be494d04b034f4',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f2_5fbuffer_5ft_3',['kasumi_f8_2_buffer_t',['../intel-ipsec-mb_8h.html#af1eb074f5da469fa655d1404be8701dd',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f3_5fbuffer_5ft_4',['kasumi_f8_3_buffer_t',['../intel-ipsec-mb_8h.html#a53ecac7e009fcbeee603f975a45d3012',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f4_5fbuffer_5ft_5',['kasumi_f8_4_buffer_t',['../intel-ipsec-mb_8h.html#accf08a616e22a2c56e6a37d56f38b354',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5fiv_5fgen_6',['kasumi_f8_iv_gen',['../intel-ipsec-mb_8h.html#a3876dca52c5c210dd0827b2a472e6d0b',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5fn_5fbuffer_5ft_7',['kasumi_f8_n_buffer_t',['../intel-ipsec-mb_8h.html#a51bbde186cdd437b6f9448e19a58e8a1',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5f1_5fbuffer_5ft_8',['kasumi_f9_1_buffer_t',['../intel-ipsec-mb_8h.html#a8b02114f003b5fe0e1ccbfdc13ac55e4',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5f1_5fbuffer_5fuser_5ft_9',['kasumi_f9_1_buffer_user_t',['../intel-ipsec-mb_8h.html#ac17147d04602964bea59322ff31ebf59',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5fiv_5fgen_10',['kasumi_f9_iv_gen',['../intel-ipsec-mb_8h.html#ac886170fc4e70b467383fdf09f0724cf',1,'intel-ipsec-mb.h']]], + ['kasumi_5finit_5ff8_5fkey_5fsched_11',['kasumi_init_f8_key_sched',['../structIMB__MGR.html#ab32d67f27cb6f06980080e09db133b24',1,'IMB_MGR']]], + ['kasumi_5finit_5ff8_5fkey_5fsched_5ft_12',['kasumi_init_f8_key_sched_t',['../intel-ipsec-mb_8h.html#a7bec62666456d8962676961cf187ac80',1,'intel-ipsec-mb.h']]], + ['kasumi_5finit_5ff9_5fkey_5fsched_13',['kasumi_init_f9_key_sched',['../structIMB__MGR.html#a58ad87104b08ba5b56fc65928677a639',1,'IMB_MGR']]], + ['kasumi_5finit_5ff9_5fkey_5fsched_5ft_14',['kasumi_init_f9_key_sched_t',['../intel-ipsec-mb_8h.html#a8b30db8462aca1c039c3fd20646f342c',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fsched_5fs_15',['kasumi_key_sched_s',['../structkasumi__key__sched__s.html',1,'']]], + ['kasumi_5fkey_5fsched_5fsize_16',['kasumi_key_sched_size',['../structIMB__MGR.html#abd0c855a3e43df48f5f21686a095f358',1,'IMB_MGR']]], + ['kasumi_5fkey_5fsched_5fsize_5ft_17',['kasumi_key_sched_size_t',['../intel-ipsec-mb_8h.html#ad7444c36024a63025e7682c16946a00a',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fsched_5ft_18',['kasumi_key_sched_t',['../intel-ipsec-mb_8h.html#a82d6866ebcf1bd46b684fe473b4b73f2',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fschedule_5fsize_19',['KASUMI_KEY_SCHEDULE_SIZE',['../intel-ipsec-mb_8h.html#a071e04977525b67c8d1accc60937eb67',1,'intel-ipsec-mb.h']]], + ['kasumi_5fuia1_20',['KASUMI_UIA1',['../structIMB__JOB.html#a11f452fba9d5cad2b2cdb929b7f244e9',1,'IMB_JOB']]], + ['key_5flen_5fin_5fbytes_21',['key_len_in_bytes',['../structIMB__JOB.html#abdd8a58a37e8d2daa1c2d333b826a65c',1,'IMB_JOB']]], + ['keyexp_5f128_22',['keyexp_128',['../structIMB__MGR.html#a7c97c5361b90abe325db55530b6f198e',1,'IMB_MGR']]], + ['keyexp_5f192_23',['keyexp_192',['../structIMB__MGR.html#a2bd7444a318de06b5f53178abf9c37ff',1,'IMB_MGR']]], + ['keyexp_5f256_24',['keyexp_256',['../structIMB__MGR.html#a431e99ddb3837a32bedb6cd426054bcc',1,'IMB_MGR']]], + ['keyexp_5ft_25',['keyexp_t',['../intel-ipsec-mb_8h.html#a60de89a391c8d6d9c51c742a87effb6f',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_c.js b/docs/search/all_c.js new file mode 100644 index 0000000000000000000000000000000000000000..85f6540f3d4afb6949ba8d150f81181454aa97bb --- /dev/null +++ b/docs/search/all_c.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['last_5fblock_5fcount_0',['last_block_count',['../structchacha20__poly1305__context__data.html#add03511d1f95376b3ac64a7916c27368',1,'chacha20_poly1305_context_data']]], + ['last_5fks_1',['last_ks',['../structchacha20__poly1305__context__data.html#afc74af4af99449f0b9f01338ae723c1a',1,'chacha20_poly1305_context_data']]], + ['len_2',['len',['../structIMB__SGL__IOV.html#a3a3662456b91011eb0585667492acc88',1,'IMB_SGL_IOV']]], + ['low_3',['low',['../structimb__uint128__t.html#a30e60cc86acb6eafaa4079c0e46ba622',1,'imb_uint128_t']]] +]; diff --git a/docs/search/all_d.js b/docs/search/all_d.js new file mode 100644 index 0000000000000000000000000000000000000000..478ff21472e2c49d9d90f09c35e51b04fe75acde --- /dev/null +++ b/docs/search/all_d.js @@ -0,0 +1,13 @@ +var searchData= +[ + ['md5_5fone_5fblock_0',['md5_one_block',['../structIMB__MGR.html#a54b5f7b625ce226ebd46a2a19e9bc724',1,'IMB_MGR']]], + ['md5_5fone_5fblock_5favx_1',['md5_one_block_avx',['../intel-ipsec-mb_8h.html#a7addeb4d0c6ddcdaa031b9e7b4dd35c2',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5favx2_2',['md5_one_block_avx2',['../intel-ipsec-mb_8h.html#a8b912812c65f4d3b592875cdd233704d',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5favx512_3',['md5_one_block_avx512',['../intel-ipsec-mb_8h.html#ac7f4da2fc0601ad50851e7e43f53e0cf',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5fsse_4',['md5_one_block_sse',['../intel-ipsec-mb_8h.html#a9d883a3a629240fe86efedd180294433',1,'intel-ipsec-mb.h']]], + ['msg_5flen_5fto_5fcipher_5fin_5fbits_5',['msg_len_to_cipher_in_bits',['../structIMB__JOB.html#a425d57ff565bb2ff76e2b0351b00c751',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fcipher_5fin_5fbytes_6',['msg_len_to_cipher_in_bytes',['../structIMB__JOB.html#a9c2e008fdb76df46a33300f692f92265',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fhash_5fin_5fbits_7',['msg_len_to_hash_in_bits',['../structIMB__JOB.html#ae45d78dcef87042ffa1daf951f715d42',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fhash_5fin_5fbytes_8',['msg_len_to_hash_in_bytes',['../structIMB__JOB.html#a40caf02dcd7e45dd91cfae33159ca26e',1,'IMB_JOB']]], + ['msk16_9',['msk16',['../structkasumi__key__sched__s.html#a091988e7975015d654ccb6904c4fdf39',1,'kasumi_key_sched_s']]] +]; diff --git a/docs/search/all_e.js b/docs/search/all_e.js new file mode 100644 index 0000000000000000000000000000000000000000..0c972f70516e1c503990d926e1dc9805f7689bdd --- /dev/null +++ b/docs/search/all_e.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['next_5fiv_0',['next_iv',['../structIMB__JOB.html#a5d473d9cb01f8e2818a6f5f0c9483bd7',1,'IMB_JOB']]], + ['next_5fjob_1',['next_job',['../structIMB__MGR.html#a106ebd60d876e56bfc6bcc03d20acce4',1,'IMB_MGR']]], + ['num_5fsgl_5fio_5fsegs_2',['num_sgl_io_segs',['../structIMB__JOB.html#a1d0a6baa8d8f7d9714fe86fe7085b8bb',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_f.js b/docs/search/all_f.js new file mode 100644 index 0000000000000000000000000000000000000000..3f451b035590120ee4dd7bd7e72f77326da5153c --- /dev/null +++ b/docs/search/all_f.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['orig_5fiv_0',['orig_IV',['../structgcm__context__data.html#a84bac4eabc12b81d26898b6fe6728a63',1,'gcm_context_data']]], + ['out_1',['out',['../structIMB__SGL__IOV.html#a54b8da76fb8a400f9f7972bdd97f6f53',1,'IMB_SGL_IOV']]] +]; diff --git a/docs/search/close.svg b/docs/search/close.svg new file mode 100644 index 0000000000000000000000000000000000000000..337d6cc13298b861a04a41f0a1003a5c9b18c8ab --- /dev/null +++ b/docs/search/close.svg @@ -0,0 +1,18 @@ + + + + + + diff --git a/docs/search/defines_2.js b/docs/search/defines_2.js new file mode 100644 index 0000000000000000000000000000000000000000..3bcf42e3f73c1331dad915fb10680753ff91cb44 --- /dev/null +++ b/docs/search/defines_2.js @@ -0,0 +1,208 @@ +var searchData= +[ + ['imb_5faes128_5fcfb_5fone_0',['IMB_AES128_CFB_ONE',['../intel-ipsec-mb_8h.html#a1e122b460a714f2981f94bf7f4bba547',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_1',['IMB_AES128_GCM_DEC',['../intel-ipsec-mb_8h.html#a2b026b2a901c76a1512eb9c8c80f9709',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5ffinalize_2',['IMB_AES128_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a280b9231c76c525a62d0ef819abd2133',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5fupdate_3',['IMB_AES128_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a2f6014cfaaa0ee5f454e2771ff6b5829',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_4',['IMB_AES128_GCM_ENC',['../intel-ipsec-mb_8h.html#adb65fc01b63a475cca4cf532ace005b6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5ffinalize_5',['IMB_AES128_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#aaf13d46f5832cf9e0cb50f7fcde020be',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5fupdate_6',['IMB_AES128_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#ab1f4c52b4145946e80ef698e469d67db',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_7',['IMB_AES128_GCM_INIT',['../intel-ipsec-mb_8h.html#ad0a8ea6b81df94d066b028ca7bf656ef',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_5fvar_5fiv_8',['IMB_AES128_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a04142f11a678c0419dff242a0ccbd2e6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fpre_9',['IMB_AES128_GCM_PRE',['../intel-ipsec-mb_8h.html#a5a442c1fcd7d427bf78e804da32bdbd1',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fprecomp_10',['IMB_AES128_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a1c63dfec645afa671f93f4fa65a6996e',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5ffinalize_11',['IMB_AES128_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#af0bd4f5841d41433339717e6b97c15ad',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5finit_12',['IMB_AES128_GMAC_INIT',['../intel-ipsec-mb_8h.html#af315c30a8cbd3c20735e22f462b47964',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5fupdate_13',['IMB_AES128_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#af0b2ba1a0f9b099a06a1ba04380b0e2a',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_14',['IMB_AES192_GCM_DEC',['../intel-ipsec-mb_8h.html#aedd2b84ebfbcb56ac5176e8793490047',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5ffinalize_15',['IMB_AES192_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a759bbe1b8494c3157f1a506c003b700b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5fupdate_16',['IMB_AES192_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#ae25c4f893fc29f09d03a8ece5fddc82f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_17',['IMB_AES192_GCM_ENC',['../intel-ipsec-mb_8h.html#af7548f17077a0b187a4ecfde4e0ff5d9',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5ffinalize_18',['IMB_AES192_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1e36a361be137a5d49b0798f054403bd',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5fupdate_19',['IMB_AES192_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a456f5bed8eb67603bdebfda230f76957',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_20',['IMB_AES192_GCM_INIT',['../intel-ipsec-mb_8h.html#a8cd402fe539de3fc8ff4062c2804096f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_5fvar_5fiv_21',['IMB_AES192_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a97791c976660a673405e6fa22ef3d262',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fpre_22',['IMB_AES192_GCM_PRE',['../intel-ipsec-mb_8h.html#a003f16aac382baff7a6b83040e58e471',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fprecomp_23',['IMB_AES192_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#ac7b3bcfd3ecbe7b2b8c7da963310b87b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5ffinalize_24',['IMB_AES192_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#a16a6f89fd4131c406d6ee7a1aeb663eb',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5finit_25',['IMB_AES192_GMAC_INIT',['../intel-ipsec-mb_8h.html#a25ed49507e66a6e98eab4a3d34e75231',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5fupdate_26',['IMB_AES192_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a113d0f4320e929f89786773f7d11c5c8',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fcfb_5fone_27',['IMB_AES256_CFB_ONE',['../intel-ipsec-mb_8h.html#ab4d68ab67a7d4a471f5cb454301aac9d',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_28',['IMB_AES256_GCM_DEC',['../intel-ipsec-mb_8h.html#a1eb77771626f83eba8482473d29ffbb1',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5ffinalize_29',['IMB_AES256_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a64d9473bc4f28c0b783baa4a31b9930e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5fupdate_30',['IMB_AES256_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a70847fd3bb5d1be79ebfa4e9104edabd',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_31',['IMB_AES256_GCM_ENC',['../intel-ipsec-mb_8h.html#a556333acdc2db1a2540a1cd1af644ca9',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5ffinalize_32',['IMB_AES256_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1f49fe7109315eb479ac1c9969e2f877',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5fupdate_33',['IMB_AES256_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a223c00c636e2552e7a3dd1a97952cc6a',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_34',['IMB_AES256_GCM_INIT',['../intel-ipsec-mb_8h.html#a8049111340bf2694799eb4e42b0a82c3',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_5fvar_5fiv_35',['IMB_AES256_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a61ddc1485781aba1969fe3ca97a22ded',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fpre_36',['IMB_AES256_GCM_PRE',['../intel-ipsec-mb_8h.html#a5771f5169d3f3af3a2419ccd8275a754',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fprecomp_37',['IMB_AES256_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a4fa417a748ca30a961334bf91297dfda',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5ffinalize_38',['IMB_AES256_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#abfbb7469f7d0240bb3d9a0e5f368b19e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5finit_39',['IMB_AES256_GMAC_INIT',['../intel-ipsec-mb_8h.html#a3d57940a54e9d94fa14dc008cd959092',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5fupdate_40',['IMB_AES256_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a91b8d98c7ffce8179ef953493a959d4b',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fblock_5fsize_41',['IMB_AES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#ab1b5db5ac58714b6f69e754a0c4ca793',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f128_42',['IMB_AES_CMAC_SUBKEY_GEN_128',['../intel-ipsec-mb_8h.html#aa1e9006a8a76ba7fb09b97816d7fdfe5',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f256_43',['IMB_AES_CMAC_SUBKEY_GEN_256',['../intel-ipsec-mb_8h.html#aca7b22139c297858794c7268439e88f0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f128_44',['IMB_AES_KEYEXP_128',['../intel-ipsec-mb_8h.html#afa111339421e794c2643df2ecf86a8c0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f192_45',['IMB_AES_KEYEXP_192',['../intel-ipsec-mb_8h.html#a574d86849e2c9ca402305eb479ca558d',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f256_46',['IMB_AES_KEYEXP_256',['../intel-ipsec-mb_8h.html#a003306124dae4920161ed6d4a7bf4e2f',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fxcbc_5fkeyexp_47',['IMB_AES_XCBC_KEYEXP',['../intel-ipsec-mb_8h.html#a5a5f2342f2b1e65e41084d915c3b2cda',1,'intel-ipsec-mb.h']]], + ['imb_5fassert_48',['IMB_ASSERT',['../intel-ipsec-mb_8h.html#af0f2f0bfad62f740bc2e22e60f48af8d',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5ffinalize_49',['IMB_CHACHA20_POLY1305_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a8f6a258b6a13952fad9c217194c716f0',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5fupdate_50',['IMB_CHACHA20_POLY1305_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a4ac8fd429d9f356003522200811b0bed',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5ffinalize_51',['IMB_CHACHA20_POLY1305_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a7575d2fafb2b970bd08a49a556602031',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5fupdate_52',['IMB_CHACHA20_POLY1305_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a2abd7cd460b99467836188d62272cd2b',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5finit_53',['IMB_CHACHA20_POLY1305_INIT',['../intel-ipsec-mb_8h.html#adb8ac3238e6dfe39a10e64602a96e643',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx_54',['IMB_CPUFLAGS_AVX',['../intel-ipsec-mb_8h.html#a0329fede890d95a97423a714410e1576',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_55',['IMB_CPUFLAGS_AVX2',['../intel-ipsec-mb_8h.html#ac0a92622a85423541536321231a6cb7d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_5ft2_56',['IMB_CPUFLAGS_AVX2_T2',['../intel-ipsec-mb_8h.html#a18f17a6bcef83c3976ca7a9b4ca624de',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_5ft3_57',['IMB_CPUFLAGS_AVX2_T3',['../intel-ipsec-mb_8h.html#a6e356586d2d3d7e93ff1b23d4a90b344',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx512_58',['IMB_CPUFLAGS_AVX512',['../intel-ipsec-mb_8h.html#a0a250932ede6ca7b69f5c763c24cf51d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx512_5ft2_59',['IMB_CPUFLAGS_AVX512_T2',['../intel-ipsec-mb_8h.html#af7bdbff703949eb2cf2dd3d3a08803de',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx_5ft2_60',['IMB_CPUFLAGS_AVX_T2',['../intel-ipsec-mb_8h.html#a85dab1abb0996e9ff4d4258fef3f9abd',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fno_5faesni_61',['IMB_CPUFLAGS_NO_AESNI',['../intel-ipsec-mb_8h.html#a6afb888ab232bab1c3ca027eb071f388',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_62',['IMB_CPUFLAGS_SSE',['../intel-ipsec-mb_8h.html#a5654f563aaad731df22c8e8517c9154c',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_5ft2_63',['IMB_CPUFLAGS_SSE_T2',['../intel-ipsec-mb_8h.html#ae6776750b801d7e177c152569c10c546',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_5ft3_64',['IMB_CPUFLAGS_SSE_T3',['../intel-ipsec-mb_8h.html#aa1cc7ac79ca44f06e3caa18fdd6d441a',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc10_5fiuup_5fdata_65',['IMB_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#ac7406ab5572b4af93ff8dbbc8ee0c961',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc11_5ffp_5fheader_66',['IMB_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#ae35b6b1b19ef6d599d47591f6c1e1732',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5ffp_5fdata_67',['IMB_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#abe747d2365eb0393ab748c218029e3c6',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5fx25_68',['IMB_CRC16_X25',['../intel-ipsec-mb_8h.html#a78217e8116876c0ecbe82fd3d417cc1c',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fa_69',['IMB_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a6ff60117e0ff260865cd80323306479b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fb_70',['IMB_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#ae07e15be0652ca8faa23c6f907d83e03',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fethernet_5ffcs_71',['IMB_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a075757c7f7b132a8f2ae13c18f36e2af',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fsctp_72',['IMB_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a86753ed15fcd8aab177d276dd4ebf169',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fwimax_5fofdma_5fdata_73',['IMB_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#ab0a9f19127f3417cc78cfdf9091af33e',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc6_5fiuup_5fheader_74',['IMB_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#ac7c45b0788456473390ca178275f9a34',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc7_5ffp_5fheader_75',['IMB_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a23fe73b1f9d40348bc98f92f859df60b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc8_5fwimax_5fofdma_5fhcs_76',['IMB_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a0edd5fa8f3351ee214f7f2a267949f6d',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fblock_5fsize_77',['IMB_DES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a95f4ece0b318cb36bdbe66089d9ef649',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkey_5fsched_5fsize_78',['IMB_DES_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a7d156b1b9d715299e08cca76e790e2b7',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkeysched_79',['IMB_DES_KEYSCHED',['../intel-ipsec-mb_8h.html#a6fbb59f06f60f1900b591735ad691ac6',1,'intel-ipsec-mb.h']]], + ['imb_5fdim_80',['IMB_DIM',['../intel-ipsec-mb_8h.html#a1beec59f9694a41c22b6b099263c070d',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5fexport_81',['IMB_DLL_EXPORT',['../intel-ipsec-mb_8h.html#a986643689f56109161597f67eb9d7b40',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5flocal_82',['IMB_DLL_LOCAL',['../intel-ipsec-mb_8h.html#a479e22216a6be5645af96c4b1433339e',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5fmin_5feth_5fpdu_5fsize_83',['IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE',['../intel-ipsec-mb_8h.html#a74dca2bb8b0c87dec90f3b985c0266af',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5ftag_5fsize_84',['IMB_DOCSIS_CRC32_TAG_SIZE',['../intel-ipsec-mb_8h.html#a59c620471ea8c9577ed593127f82c758',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5faesni_85',['IMB_FEATURE_AESNI',['../intel-ipsec-mb_8h.html#a19c147087d2ae918baa22f90fa9b3b05',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5faesni_5femu_86',['IMB_FEATURE_AESNI_EMU',['../intel-ipsec-mb_8h.html#a421158db6d19c97a6bceb32eb4a839ed',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx_87',['IMB_FEATURE_AVX',['../intel-ipsec-mb_8h.html#a7af992d12b4f9a6e789384969a066056',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx2_88',['IMB_FEATURE_AVX2',['../intel-ipsec-mb_8h.html#a225e2cd9c1445795fc62879eabad2530',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fifma_89',['IMB_FEATURE_AVX512_IFMA',['../intel-ipsec-mb_8h.html#a0134dfaffe12427a8f2182247c6f1f48',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fskx_90',['IMB_FEATURE_AVX512_SKX',['../intel-ipsec-mb_8h.html#ae6d6e3427c81b278d64f0189401c4964',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512bw_91',['IMB_FEATURE_AVX512BW',['../intel-ipsec-mb_8h.html#a00caae81d425acb6b946bebb25ac267f',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512cd_92',['IMB_FEATURE_AVX512CD',['../intel-ipsec-mb_8h.html#abbdb390b80ff1a92f31a7a9adc9beeb9',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512dq_93',['IMB_FEATURE_AVX512DQ',['../intel-ipsec-mb_8h.html#a8ca2c970faa0f62a3f4a6b6c18432bb1',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512f_94',['IMB_FEATURE_AVX512F',['../intel-ipsec-mb_8h.html#ab673c08624d868ca0dd89e1a9ec28f00',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512vl_95',['IMB_FEATURE_AVX512VL',['../intel-ipsec-mb_8h.html#a803871900c08e2dc1b3f805ed4ecefd8',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx_5fifma_96',['IMB_FEATURE_AVX_IFMA',['../intel-ipsec-mb_8h.html#a8a946f162ff5e363d04f8b44d2f48747',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fbmi2_97',['IMB_FEATURE_BMI2',['../intel-ipsec-mb_8h.html#a557f3de4bcc0bd7e9bd90e5e1d1b9b09',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fcmov_98',['IMB_FEATURE_CMOV',['../intel-ipsec-mb_8h.html#a5eef394935c651931b1cf560f31c841c',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fgfni_99',['IMB_FEATURE_GFNI',['../intel-ipsec-mb_8h.html#aa67ab4088ac2f8cadbe839101d276567',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fpclmulqdq_100',['IMB_FEATURE_PCLMULQDQ',['../intel-ipsec-mb_8h.html#a1af54a4a862936a6793af2ff44163ba5',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fdata_101',['IMB_FEATURE_SAFE_DATA',['../intel-ipsec-mb_8h.html#abd58fdbe1f1ef53cf509a0b37ba8aa4b',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fparam_102',['IMB_FEATURE_SAFE_PARAM',['../intel-ipsec-mb_8h.html#a4e9506dc72ab4e6a2af463238ccb3080',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fself_5ftest_103',['IMB_FEATURE_SELF_TEST',['../intel-ipsec-mb_8h.html#a8b4181547ded3ff17859d2632c468151',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fself_5ftest_5fpass_104',['IMB_FEATURE_SELF_TEST_PASS',['../intel-ipsec-mb_8h.html#a673b969c5f01579f6ca0a6b003cf1197',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fshani_105',['IMB_FEATURE_SHANI',['../intel-ipsec-mb_8h.html#acb9c15eabfb9bafeb15a06886fa91e90',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsse4_5f2_106',['IMB_FEATURE_SSE4_2',['../intel-ipsec-mb_8h.html#abcc243fc505aca3eac6963eff25e9b99',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvaes_107',['IMB_FEATURE_VAES',['../intel-ipsec-mb_8h.html#af4321e4e3c947f9b9fb7f204aaecf2c3',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvpclmulqdq_108',['IMB_FEATURE_VPCLMULQDQ',['../intel-ipsec-mb_8h.html#a153d695ecd1af69a0d23385a50654614',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5faesni_5foff_109',['IMB_FLAG_AESNI_OFF',['../intel-ipsec-mb_8h.html#a4c53fb8e39bbc6e3f39d0558934e5d55',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5fgfni_5foff_110',['IMB_FLAG_GFNI_OFF',['../intel-ipsec-mb_8h.html#ad3faac9d03a87621ae02c10337b4edf8',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5fshani_5foff_111',['IMB_FLAG_SHANI_OFF',['../intel-ipsec-mb_8h.html#a3d45fcff99a9ea952b4dd6ef70e36e0c',1,'intel-ipsec-mb.h']]], + ['imb_5fflush_5fburst_112',['IMB_FLUSH_BURST',['../intel-ipsec-mb_8h.html#afff52a9aaa1c1875e7c905e2fb29fc43',1,'intel-ipsec-mb.h']]], + ['imb_5fflush_5fjob_113',['IMB_FLUSH_JOB',['../intel-ipsec-mb_8h.html#a4726e3683b3579b1e643134278386d72',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f128_5fkey_5flen_114',['IMB_GCM_128_KEY_LEN',['../intel-ipsec-mb_8h.html#ae97681c203afbd818a39fb797613a232',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f192_5fkey_5flen_115',['IMB_GCM_192_KEY_LEN',['../intel-ipsec-mb_8h.html#a7201a0be67f1e6f3757169b0166dd47d',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f256_5fkey_5flen_116',['IMB_GCM_256_KEY_LEN',['../intel-ipsec-mb_8h.html#a4c4c40ec87c2354832f5b182b22d2111',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fblock_5flen_117',['IMB_GCM_BLOCK_LEN',['../intel-ipsec-mb_8h.html#a59def6ed420a4ef91ce4a6562cc2c4d5',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fenc_5fkey_5flen_118',['IMB_GCM_ENC_KEY_LEN',['../intel-ipsec-mb_8h.html#a0c6f54ab6663187fa401c766a981a41e',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fiv_5fdata_5flen_119',['IMB_GCM_IV_DATA_LEN',['../intel-ipsec-mb_8h.html#a87884da318b3013893b725d047b53702',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fkey_5fsets_120',['IMB_GCM_KEY_SETS',['../intel-ipsec-mb_8h.html#a73715b8c1ba918ce6d5c21749841b47b',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fcompleted_5fjob_121',['IMB_GET_COMPLETED_JOB',['../intel-ipsec-mb_8h.html#a18e5a7f85573f0006448deda41014117',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fnext_5fburst_122',['IMB_GET_NEXT_BURST',['../intel-ipsec-mb_8h.html#a6029af3f88d1908dd11a8f3e9eb9fd70',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fnext_5fjob_123',['IMB_GET_NEXT_JOB',['../intel-ipsec-mb_8h.html#a34b6006fc39dd4ea23d3fec49237d382',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_124',['IMB_GHASH',['../intel-ipsec-mb_8h.html#a477ad08a4c55e2c6458b7e86dae803ec',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_5fpre_125',['IMB_GHASH_PRE',['../intel-ipsec-mb_8h.html#a230f16784519d7e2b69de87871d35cb2',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f32_126',['IMB_HEC_32',['../intel-ipsec-mb_8h.html#a37996cede66ee8036b1b113af5f553ac',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f64_127',['IMB_HEC_64',['../intel-ipsec-mb_8h.html#aa0b62ba2c9fb07a1ad3c8105ec4374a4',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fblock_5fsize_128',['IMB_KASUMI_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#aa31ca303e03fc548199f95e5335889e1',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fdigest_5fsize_129',['IMB_KASUMI_DIGEST_SIZE',['../intel-ipsec-mb_8h.html#a56bd44d01c91469b57efa36f360a83bf',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_130',['IMB_KASUMI_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a45ac331e1f9241e53fc35a1f88dec6ce',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_5fbit_131',['IMB_KASUMI_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#ae3befb703f7dec2c0695e61542f3212c',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f2_5fbuffer_132',['IMB_KASUMI_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a884bfa72fc232c7e84043d0faee719ee',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f3_5fbuffer_133',['IMB_KASUMI_F8_3_BUFFER',['../intel-ipsec-mb_8h.html#af35e87948218048e4bbad7162208fe41',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f4_5fbuffer_134',['IMB_KASUMI_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a7ed6be7362e5ff7553fb4e7633bee76a',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5fn_5fbuffer_135',['IMB_KASUMI_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a65b1afbadee23111046746a4d302bb0d',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_136',['IMB_KASUMI_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#a43c13bbdf83e2c75d6dd29d01dec4a1f',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_5fuser_137',['IMB_KASUMI_F9_1_BUFFER_USER',['../intel-ipsec-mb_8h.html#a66dcf1985c046ef619141a6706159940',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff8_5fkey_5fsched_138',['IMB_KASUMI_INIT_F8_KEY_SCHED',['../intel-ipsec-mb_8h.html#aafcc540a047a6036adf4da0613ea5272',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff9_5fkey_5fsched_139',['IMB_KASUMI_INIT_F9_KEY_SCHED',['../intel-ipsec-mb_8h.html#a80a71ab95fd99c8427e171e45cebbd92',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fiv_5fsize_140',['IMB_KASUMI_IV_SIZE',['../intel-ipsec-mb_8h.html#adf7e13e43c6bb30489d0e26f11e39440',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsched_5fsize_141',['IMB_KASUMI_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#ae3812840fbe6b9b5ea5e84a6b86a7ed3',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsize_142',['IMB_KASUMI_KEY_SIZE',['../intel-ipsec-mb_8h.html#a46d59512e3a7254e54018181b7a31102',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5fburst_5fsize_143',['IMB_MAX_BURST_SIZE',['../intel-ipsec-mb_8h.html#af9070daf9749490016e9b5106b338867',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5fjobs_144',['IMB_MAX_JOBS',['../intel-ipsec-mb_8h.html#ab12d22470a68b5d3852c8dd972784d97',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5ftag_5flen_145',['IMB_MAX_TAG_LEN',['../intel-ipsec-mb_8h.html#a41087ea6dce2e096ebb02a613600c2ed',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fblock_5fsize_146',['IMB_MD5_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a567100ef1c32f08b9e75e1fb378171a3',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fdigest_5fsize_5fin_5fbytes_147',['IMB_MD5_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a472528903f010fe2c365d1ff23565eaa',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fone_5fblock_148',['IMB_MD5_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a52b27293e2a19809c5f7d0f2afab97a2',1,'intel-ipsec-mb.h']]], + ['imb_5fqueue_5fsize_149',['IMB_QUEUE_SIZE',['../intel-ipsec-mb_8h.html#a712f6e7ecb2a16f8e1bc96b84d188365',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_150',['IMB_SHA1',['../intel-ipsec-mb_8h.html#ae61900d14cde2836621a1421141548a4',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fblock_5fsize_151',['IMB_SHA1_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#af19fd9d6768e33720cab4ecd9aabad91',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fdigest_5fsize_5fin_5fbytes_152',['IMB_SHA1_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a8c821d8cf5520f6f46e9b835eeeb2066',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fone_5fblock_153',['IMB_SHA1_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ac2f9695b12c32c303faa7c93c1c24c9b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_154',['IMB_SHA224',['../intel-ipsec-mb_8h.html#a542a40ced46bf97a678c6b44c325b360',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fdigest_5fsize_5fin_5fbytes_155',['IMB_SHA224_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a836641341e86b80f67716eabdf4a64c5',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fone_5fblock_156',['IMB_SHA224_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ab1f7eb5e9bcd0c69a70dc76747f919f6',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_157',['IMB_SHA256',['../intel-ipsec-mb_8h.html#a37338b9f91315e5de0c4f3d19859aa7b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fdigest_5fsize_5fin_5fbytes_158',['IMB_SHA256_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a4bc9eee15ba0ce09f545af8916292731',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fone_5fblock_159',['IMB_SHA256_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a6023b87b5d1c6dc7dc0547db540af997',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_160',['IMB_SHA384',['../intel-ipsec-mb_8h.html#ac8052b8b996f866f27b4d443bcda0d5f',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fdigest_5fsize_5fin_5fbytes_161',['IMB_SHA384_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a2538a5e82141eb5a54f78660241e186c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fone_5fblock_162',['IMB_SHA384_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a763d0cfb16263906e44f46696b16124b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_163',['IMB_SHA512',['../intel-ipsec-mb_8h.html#ac806d6b629633e801409d3e363d83f6c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fdigest_5fsize_5fin_5fbytes_164',['IMB_SHA512_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a9efcf370f0db0b21f7cf4a3b72eec53e',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fone_5fblock_165',['IMB_SHA512_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a71d0a529b355a434efeeda2e01870ba9',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f224_5fblock_5fsize_166',['IMB_SHA_224_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a7c912b7543122de6f05355fa4848eb83',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f256_5fblock_5fsize_167',['IMB_SHA_256_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a7eed0159b050f8d78cddcc78b4b941ef',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f384_5fblock_5fsize_168',['IMB_SHA_384_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a256cc0f9eaba38030ca3f7589e479855',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f512_5fblock_5fsize_169',['IMB_SHA_512_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a4141433b17ade98e14a585e0409cb62d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_170',['IMB_SNOW3G_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a4bcfbb70da38433f8bcd3468779026f1',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_5fbit_171',['IMB_SNOW3G_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#a5d7a9e694b932b1c198782e22dc1ead5',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f2_5fbuffer_172',['IMB_SNOW3G_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a51085837f0775feae26fe3885f7459b7',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f4_5fbuffer_173',['IMB_SNOW3G_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a79b873457a74ad6c50e689709d79a0fb',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_174',['IMB_SNOW3G_F8_8_BUFFER',['../intel-ipsec-mb_8h.html#a8536e68d053999123eb104762ef51932',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_5fmultikey_175',['IMB_SNOW3G_F8_8_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#aca8fb428a3b291950cd9dc6277ad801a',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_176',['IMB_SNOW3G_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a4538b34eaf3a3d402041ada1e8d03e2c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_5fmultikey_177',['IMB_SNOW3G_F8_N_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#a4a5ae1b04c7aee099c2f8fcbd488c38c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff9_5f1_5fbuffer_178',['IMB_SNOW3G_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#aa6a951033156fc06c25afb4f79de8851',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5finit_5fkey_5fsched_179',['IMB_SNOW3G_INIT_KEY_SCHED',['../intel-ipsec-mb_8h.html#ac5b5ff1cc9cf44ea8587a7d054b7459d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5fkey_5fsched_5fsize_180',['IMB_SNOW3G_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a1ddd766994cb5d0b8e9ef5cd580cdcd2',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_181',['IMB_SUBMIT_BURST',['../intel-ipsec-mb_8h.html#aabfad78b032b6b160e3ba51477132524',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_5fnocheck_182',['IMB_SUBMIT_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#aa06f0e18794d4a95d16e61dfb1af70c4',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_183',['IMB_SUBMIT_CIPHER_BURST',['../intel-ipsec-mb_8h.html#ae487720f5112a3ecec659ab4aeada77e',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_5fnocheck_184',['IMB_SUBMIT_CIPHER_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#af819e81f584b3b56e0201a7174b3c0eb',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_185',['IMB_SUBMIT_HASH_BURST',['../intel-ipsec-mb_8h.html#a3702b37004e38dd27a9b8a008a0c48ed',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_5fnocheck_186',['IMB_SUBMIT_HASH_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#afd6b80655a6129fe063d2ffb391e3015',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_187',['IMB_SUBMIT_JOB',['../intel-ipsec-mb_8h.html#a3602a4ccaf1da4bf6787169370cef04f',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_5fnocheck_188',['IMB_SUBMIT_JOB_NOCHECK',['../intel-ipsec-mb_8h.html#af23c64e466272c5ae90fcf78fbcd0c3d',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_189',['IMB_VERSION',['../intel-ipsec-mb_8h.html#ad9bcb21181d1e07b4f66d8bf0514c1a3',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fnum_190',['IMB_VERSION_NUM',['../intel-ipsec-mb_8h.html#a415abfc6067c0b1663a508993a4eb0d4',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fstr_191',['IMB_VERSION_STR',['../intel-ipsec-mb_8h.html#a874972638400dafdb542319d61bfbfbc',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fdigest_5flen_5fin_5fbytes_5fmax_192',['IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX',['../intel-ipsec-mb_8h.html#a2b1faea7079fa3ddb2cabeb41aeb2a0a',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fdigest_5flen_5fin_5fbytes_5fmin_193',['IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN',['../intel-ipsec-mb_8h.html#a3dc24506ef16ad090414608c762b3d3c',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fiv_5flen_5fin_5fbytes_5fmax_194',['IMB_ZUC256_IV_LEN_IN_BYTES_MAX',['../intel-ipsec-mb_8h.html#a5ab93cc1831b961f3d04ff913f63b832',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fiv_5flen_5fin_5fbytes_5fmin_195',['IMB_ZUC256_IV_LEN_IN_BYTES_MIN',['../intel-ipsec-mb_8h.html#a18e1733291bb917240ef117f52c6aa6c',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc256_5fkey_5flen_5fin_5fbytes_196',['IMB_ZUC256_KEY_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#a35d04f4dc107531e37fcab1fd5efbf93',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5fdigest_5flen_5fin_5fbytes_197',['IMB_ZUC_DIGEST_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#adbb9f3bfdc1d416c20a579eb3a8b0b8f',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f1_5fbuffer_198',['IMB_ZUC_EEA3_1_BUFFER',['../intel-ipsec-mb_8h.html#adb23f957dde69fb3bb4c6798d1a69da4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f4_5fbuffer_199',['IMB_ZUC_EEA3_4_BUFFER',['../intel-ipsec-mb_8h.html#a57445dd863838d04cf44f4d79c4dbcaf',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5fn_5fbuffer_200',['IMB_ZUC_EEA3_N_BUFFER',['../intel-ipsec-mb_8h.html#a6382e3185f31e61f995ab90110c52991',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5f1_5fbuffer_201',['IMB_ZUC_EIA3_1_BUFFER',['../intel-ipsec-mb_8h.html#ac789fe54f3a8f92144831a67b14501d4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5fn_5fbuffer_202',['IMB_ZUC_EIA3_N_BUFFER',['../intel-ipsec-mb_8h.html#ae840d62f1246c0c4270c3319ef356d3c',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5fiv_5flen_5fin_5fbytes_203',['IMB_ZUC_IV_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#aaacb0749fd75d1133414685b5694652d',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5fkey_5flen_5fin_5fbytes_204',['IMB_ZUC_KEY_LEN_IN_BYTES',['../intel-ipsec-mb_8h.html#a053b0646f055d7ee3be974bb2bfe534b',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/enumvalues_0.js b/docs/search/enumvalues_0.js new file mode 100644 index 0000000000000000000000000000000000000000..c6dd172d3308a27ac5c5ec55cca0f69d392424dc --- /dev/null +++ b/docs/search/enumvalues_0.js @@ -0,0 +1,155 @@ +var searchData= +[ + ['imb_5farch_5favx_0',['IMB_ARCH_AVX',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6aa831f565e051aa13fcad5b224358e57d',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx2_1',['IMB_ARCH_AVX2',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a701628fddfdd84d56495839f20573de6',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx512_2',['IMB_ARCH_AVX512',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1df4f12c5769b52a95f1a4b89a8163c8',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnoaesni_3',['IMB_ARCH_NOAESNI',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a6a5b29422391adbb6806575bf7ccae44',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnone_4',['IMB_ARCH_NONE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1ff5121849f183accf83fd652b854733',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnum_5',['IMB_ARCH_NUM',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6ae6536a65a8d0cd53faa325d5ee4dcdda',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fsse_6',['IMB_ARCH_SSE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a3c2ccbe898a62cbaed5bad7717fdfa16',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fccm_7',['IMB_AUTH_AES_CCM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a824187b143cb8d4b1778d10bcedbce3e',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_8',['IMB_AUTH_AES_CMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aed85279dbff19be59c16216364375a48',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5f256_9',['IMB_AUTH_AES_CMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a0d0cad9513b19110454c9e4e9c549205',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5fbitlen_10',['IMB_AUTH_AES_CMAC_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa9a94b55feed30ce57ecadf17b7c1a83',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_11',['IMB_AUTH_AES_GMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa1496e45f00afe54aeeb527089e14a0d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f128_12',['IMB_AUTH_AES_GMAC_128',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a189af960a1558b7ac3289c0892739ea0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f192_13',['IMB_AUTH_AES_GMAC_192',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adfd8db9df85ef929d9e4a38612f07d66',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f256_14',['IMB_AUTH_AES_GMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae6fab01dbaf8b0110ca8ccdce98547af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fxcbc_15',['IMB_AUTH_AES_XCBC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a17deb61684701b7ec37f68bd34b6d454',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_16',['IMB_AUTH_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab737fa5ff7244ddf5024905f6477ae01',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_5fsgl_17',['IMB_AUTH_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a71119edfbe668d423e910b11c0efb712',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc10_5fiuup_5fdata_18',['IMB_AUTH_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ac27de87b2687b1e2a293544be6d3fab9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc11_5ffp_5fheader_19',['IMB_AUTH_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab0f02ac14a03f137a83bb7d6dff539fc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5ffp_5fdata_20',['IMB_AUTH_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa78e695260f8fe6bb56d242771624ccc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5fx25_21',['IMB_AUTH_CRC16_X25',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a22e2e766cbec475508af01c6a722c2c3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fa_22',['IMB_AUTH_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3a8aa07fe9f70ece22f9351db0f8bb31',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fb_23',['IMB_AUTH_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a77ccdf9abd97310f6808c8cf12c1a395',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fethernet_5ffcs_24',['IMB_AUTH_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a421294f9a35f87e796f96b48257e6dc2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fsctp_25',['IMB_AUTH_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff8b335b5391fc33c80c076590de44',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fwimax_5fofdma_5fdata_26',['IMB_AUTH_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad3936131d6804e6c6aa3bd497ae675ab',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc6_5fiuup_5fheader_27',['IMB_AUTH_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a88175070e077037332ad5cd5fdece46f',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc7_5ffp_5fheader_28',['IMB_AUTH_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab7f1372ade9da040caf195b13f2b79f0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc8_5fwimax_5fofdma_5fhcs_29',['IMB_AUTH_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae62fe411b683840a5f25c199b66c7f42',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcustom_30',['IMB_AUTH_CUSTOM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a70151af8840242c8f5bdb375f2f1135b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fdocsis_5fcrc32_31',['IMB_AUTH_DOCSIS_CRC32',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae006c86697198ca852fc612d376135dc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fgcm_5fsgl_32',['IMB_AUTH_GCM_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ada7d3fc165b6855ae3e83467de9916b2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fghash_33',['IMB_AUTH_GHASH',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a67297eacd4913bd65a1aae650c3a22e3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f1_34',['IMB_AUTH_HMAC_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a441cc344587709b2ff2fc26f36ff1e58',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f224_35',['IMB_AUTH_HMAC_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad0ad42ef3cb8cef38a29bcc667e70fca',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f256_36',['IMB_AUTH_HMAC_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a60c1e71108516903aded4534f0f2e995',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f384_37',['IMB_AUTH_HMAC_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae98f8f8e42b7a026bb2aa2a402a291b9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f512_38',['IMB_AUTH_HMAC_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad8f7bc68ad10fe75665692a27ada55f1',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fkasumi_5fuia1_39',['IMB_AUTH_KASUMI_UIA1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a42dc1dc990059b61ef784813e0f8f49a',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fmd5_40',['IMB_AUTH_MD5',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a958fb278e4745955ebc791ba652954fe',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnull_41',['IMB_AUTH_NULL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a25f47297460775ab891a0abb8240b8e0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnum_42',['IMB_AUTH_NUM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a5d34120f8a1c5cde857baf32536868af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpoly1305_43',['IMB_AUTH_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a93e6c082f7ec5926a087d139a2bd2e56',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpon_5fcrc_5fbip_44',['IMB_AUTH_PON_CRC_BIP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a2fcc080535b4dd16502ecd2b01c14048',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f1_45',['IMB_AUTH_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adcaf4a43c9c114bcdb75ff44ba1bea77',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f224_46',['IMB_AUTH_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3936068075b4572d859c9b1984312325',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f256_47',['IMB_AUTH_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4962dbdf6fcb8d743e4ab55ce23c4b88',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f384_48',['IMB_AUTH_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006afe74859581d19e70b8667773662081bc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f512_49',['IMB_AUTH_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa12aca10078adbe0bc2a021190663e2d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow3g_5fuia2_5fbitlen_50',['IMB_AUTH_SNOW3G_UIA2_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4eadf3270a81eb2567388843da2fa89b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow_5fv_5faead_51',['IMB_AUTH_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff331c62b01fb8427e63e18fc433f6',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc256_5feia3_5fbitlen_52',['IMB_AUTH_ZUC256_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a7c535075abfb0878168a00600a762fe0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc_5feia3_5fbitlen_53',['IMB_AUTH_ZUC_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a188c2297dc5228820e690e8938cd407b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbc_54',['IMB_CIPHER_CBC',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0e2ebfcfd101ab4a39129082b2de517f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbcs_5f1_5f9_55',['IMB_CIPHER_CBCS_1_9',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca71d87659793f7377d1d8b3315a3559f8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fccm_56',['IMB_CIPHER_CCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca7d3946d77e4364bc45a9e72499bf817f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_57',['IMB_CIPHER_CHACHA20',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca39d8d7ebca424a1c1a65bc04b5f64741',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_58',['IMB_CIPHER_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caa5c375f97dce274a2446c8abf90aa61a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_5fsgl_59',['IMB_CIPHER_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca40926d8b731a401ec3e86ffa2089a1eb',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_60',['IMB_CIPHER_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca5b4126e439eaff0975846425594201ea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_5fbitlen_61',['IMB_CIPHER_CNTR_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca6f69815480531651948b5549fa84023a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcustom_62',['IMB_CIPHER_CUSTOM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca4301726c71a4faf412a1a011cc4f0df8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes_63',['IMB_CIPHER_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca049e85047492494fac0a4ec785707851',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes3_64',['IMB_CIPHER_DES3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca639b1382de364a2b810f7876189bc64f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fdes_65',['IMB_CIPHER_DOCSIS_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caf9d62e1ff3a1a8b2ed1dcc515a3ffe1b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fsec_5fbpi_66',['IMB_CIPHER_DOCSIS_SEC_BPI',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caed8ba68d7d2d7602407f17305d1d99b7',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fecb_67',['IMB_CIPHER_ECB',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca357dec895f8dfa3371add1feaa001f56',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_68',['IMB_CIPHER_GCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0f97344e525fef167a92d1a59a0492aa',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_5fsgl_69',['IMB_CIPHER_GCM_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca48e024bc875c46b714b19a1c19908918',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fkasumi_5fuea1_5fbitlen_70',['IMB_CIPHER_KASUMI_UEA1_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3f7a3edb272955efea015c5bb8572e3b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnull_71',['IMB_CIPHER_NULL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0bebd24d7311dedce100455dd3a8ddf6',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnum_72',['IMB_CIPHER_NUM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca78c470d405fb36037e41224b99d23c75',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fpon_5faes_5fcntr_73',['IMB_CIPHER_PON_AES_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca896285cd2fe56f73011b908e490cbbea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow3g_5fuea2_5fbitlen_74',['IMB_CIPHER_SNOW3G_UEA2_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca88c3ff9a6816b3629dc5b4815b47607e',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_75',['IMB_CIPHER_SNOW_V',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cafea56410df85a0b006963e50fba78465',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_5faead_76',['IMB_CIPHER_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cada26994f8467300f862dbf45b5a22409',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fzuc_5feea3_77',['IMB_CIPHER_ZUC_EEA3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3618bfaab3c19aabc8a93a03ae121efc',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fdecrypt_78',['IMB_DIR_DECRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eea881652011955c227d94a881d2d2c5c59',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fencrypt_79',['IMB_DIR_ENCRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eeabed12e936823e8f8b2de70388a2d15ce',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5faad_5flen_80',['IMB_ERR_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0ad3ade3fe22c5887d9bf444ff3923aa',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5flen_81',['IMB_ERR_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafe5af1c17e1caece14b3a5f659a1336d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5ftag_5flen_82',['IMB_ERR_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6af5bd8fb7690b46afe3d2c1e623808e',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fburst_5fooo_83',['IMB_ERR_BURST_OOO',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae4584575cf7134ba02ca1d38bb3f6444',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fburst_5fsize_84',['IMB_ERR_BURST_SIZE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae2bc238b157942636836eaf0eedb5ddf',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fburst_5fsuite_5fid_85',['IMB_ERR_BURST_SUITE_ID',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba436eb6f9e045a02e3ad5998da8c13d78',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5flen_86',['IMB_ERR_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bab5a2b4c1df7778c3efc236976062ed94',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5fmode_87',['IMB_ERR_CIPH_MODE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba61a411feda0e8cf5aff8c58ad2e92bdd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fhash_5falgo_88',['IMB_ERR_HASH_ALGO',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8ac2802661bd9928d0c766129db2a814',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fiv_5flen_89',['IMB_ERR_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baac8500b58709f0bd05ad129337bb6af2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5faad_5flen_90',['IMB_ERR_JOB_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba31a6b672b95d7d100a97fa6d788c0dd6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5flen_91',['IMB_ERR_JOB_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69dd17473dc4afb2f04c417fe6882608',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5ftag_5flen_92',['IMB_ERR_JOB_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1b7e4c43808275e9cd63ca21091e5ab2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fchain_5forder_93',['IMB_ERR_JOB_CHAIN_ORDER',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacaec18a061b4bf792178408d69775950',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5fdir_94',['IMB_ERR_JOB_CIPH_DIR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacbf5d148be9963df1d46400ff26396db',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5flen_95',['IMB_ERR_JOB_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae1c9e58d4be3fdd7bb37247667472920',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fiv_5flen_96',['IMB_ERR_JOB_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba390f91827f87bbd3233ad6e1f104b2b4',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fkey_5flen_97',['IMB_ERR_JOB_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badfea107da94936d30900f23ad22b5285',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5faad_98',['IMB_ERR_JOB_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3ef3d3180e3052862c796cf89320d60c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_99',['IMB_ERR_JOB_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba503544043c60c2195003c00b18141beb',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_5fkey_100',['IMB_ERR_JOB_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba97fb89f93299f67de86aac919bb83aa9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fdst_101',['IMB_ERR_JOB_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba55b3ddc1a48cb14ad020031b8d5b4927',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fghash_5finit_5ftag_102',['IMB_ERR_JOB_NULL_GHASH_INIT_TAG',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6c97704719f286b14204ca144bbdc7a9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fipad_103',['IMB_ERR_JOB_NULL_HMAC_IPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba15f55e87bb42cf5494dd01b9a1c4d712',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fopad_104',['IMB_ERR_JOB_NULL_HMAC_OPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacb9286af7533993ab79982b288aab97a',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fiv_105',['IMB_ERR_JOB_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0d35070d6476cde0202deb5531ae09c2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fkey_106',['IMB_ERR_JOB_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba22402460340d6b34492ece390a7913c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fnext_5fiv_107',['IMB_ERR_JOB_NULL_NEXT_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba983561202b1623b0bd1f691bd6036039',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsgl_5fctx_108',['IMB_ERR_JOB_NULL_SGL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba05f9212af539a22ad65b12a9abc8e5c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsrc_109',['IMB_ERR_JOB_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4bc6a784afa9bb88c5afe4fd1b5f610d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk1_5fexp_110',['IMB_ERR_JOB_NULL_XCBC_K1_EXP',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69bf9e54fdf84b154284eab071c98804',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk2_111',['IMB_ERR_JOB_NULL_XCBC_K2',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bad7fbd09092821fb8605e11384dcdc428',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk3_112',['IMB_ERR_JOB_NULL_XCBC_K3',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baf12955208a99c8feeeb580fee940b7c3',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fpon_5fpli_113',['IMB_ERR_JOB_PON_PLI',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bace7e834842ddef5b063c99544d00b303',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fsrc_5foffset_114',['IMB_ERR_JOB_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba207979780b5d8472dfef2f5e12998d22',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fkey_5flen_115',['IMB_ERR_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8d6b3ea721216bb12fbe40e72a5bd768',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmax_116',['IMB_ERR_MAX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bac773be615f9904fd69e961451fb0fac6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmin_117',['IMB_ERR_MIN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3f9d55f356a0caf413d2114a468ba659',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmissing_5fcpuflags_5finit_5fmgr_118',['IMB_ERR_MISSING_CPUFLAGS_INIT_MGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0babdd0b7bc92e7dcf166d36a9bcd118d60',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fno_5faesni_5femu_119',['IMB_ERR_NO_AESNI_EMU',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4496bf14e2ddf1133a384f9fbf96d5dc',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5faad_120',['IMB_ERR_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badc7d17d90012d5c14fd8a12d9ea12543',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_121',['IMB_ERR_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1e20954ec8a501093175d640631550c1',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_5fkey_122',['IMB_ERR_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baa4f5aeab78ed40a4c19389b8f691306c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fburst_123',['IMB_ERR_NULL_BURST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba99d845a176cd571dd6f99b0a430291c6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fctx_124',['IMB_ERR_NULL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0b57ec134f6e10e11c3fbcf77ede282d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fdst_125',['IMB_ERR_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1a61dcc99a1a9725c89076dec3c02e02',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fexp_5fkey_126',['IMB_ERR_NULL_EXP_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3b60a570372529486da69c6d2b7be382',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fiv_127',['IMB_ERR_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafcfe882ee9730fd1724b587106f2f419',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fjob_128',['IMB_ERR_NULL_JOB',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba007c436f8e21f317e8f9225c5b009d38',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fkey_129',['IMB_ERR_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba63029e3673eef74345ad80867f7ba0ef',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fmbmgr_130',['IMB_ERR_NULL_MBMGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba9ba69cc32a2d17e3e7b4d4c982f26abd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fsrc_131',['IMB_ERR_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba308819b7cbdadfd6bbee290408880302',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fqueue_5fspace_132',['IMB_ERR_QUEUE_SPACE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4f73ed8232981b84df1fee21024e01ac',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fselftest_133',['IMB_ERR_SELFTEST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1873d38b8aaf5ebcd3f542b15c418829',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fsrc_5foffset_134',['IMB_ERR_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba20dfbdc8feb8332a9efe51de48fa6e09',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f128_5fbytes_135',['IMB_KEY_128_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaf3e82db9a8b5b1b229c201735938969d',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f192_5fbytes_136',['IMB_KEY_192_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cca573d62aa6a04d9ce2b3c34e226846598',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f256_5fbytes_137',['IMB_KEY_256_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccac139e7555c9b30725359698833eb213a',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f64_5fbytes_138',['IMB_KEY_64_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaa1eb9eb83fca5f5dd6578f27733eb390',1,'intel-ipsec-mb.h']]], + ['imb_5forder_5fcipher_5fhash_139',['IMB_ORDER_CIPHER_HASH',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5e3ab845752ce3d16e40ea7802ce4a52',1,'intel-ipsec-mb.h']]], + ['imb_5forder_5fhash_5fcipher_140',['IMB_ORDER_HASH_CIPHER',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5de8d032616267941d99dad14801044d',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fall_141',['IMB_SGL_ALL',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da2f0678ab7ad9bb30c040e0b3ee2fa408',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fcomplete_142',['IMB_SGL_COMPLETE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3dab7a602903c0fbaad7ffb893cb2547975',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5finit_143',['IMB_SGL_INIT',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3daf6e77a48759ab366cc1160725ff4261e',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fupdate_144',['IMB_SGL_UPDATE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da15c5d0aeeb9d5eb4e4c4878c837c5977',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fbeing_5fprocessed_145',['IMB_STATUS_BEING_PROCESSED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaa562b01ade81e0196fc23cd79d534141',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_146',['IMB_STATUS_COMPLETED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba508a9f195acee09621542ca687ba2341',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fauth_147',['IMB_STATUS_COMPLETED_AUTH',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba79df83a15cdafc2a6c79751457ae9792',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fcipher_148',['IMB_STATUS_COMPLETED_CIPHER',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba6fcaccd3853c9e8cf3b402edc3e22089',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5ferror_149',['IMB_STATUS_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba0ef40895610ac6343d171f65be676a1d',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finternal_5ferror_150',['IMB_STATUS_INTERNAL_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba546465b9d4cfb1f9c98c864bf5ed51fb',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finvalid_5fargs_151',['IMB_STATUS_INVALID_ARGS',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaee85b6a8bc33e0b39001e9ff538540b7',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_0.js b/docs/search/functions_0.js new file mode 100644 index 0000000000000000000000000000000000000000..b82d1166499d4d42c12504c02adb6f1bb45c8705 --- /dev/null +++ b/docs/search/functions_0.js @@ -0,0 +1,117 @@ +var searchData= +[ + ['aes_5fcmac_5fsubkey_5fgen_5favx_0',['aes_cmac_subkey_gen_avx',['../intel-ipsec-mb_8h.html#a6541825066a8d2b8b3c7b6e2c4b28c83',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx2_1',['aes_cmac_subkey_gen_avx2',['../intel-ipsec-mb_8h.html#a6903e1e0dfb8fd75c2f556596f3efa1b',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx512_2',['aes_cmac_subkey_gen_avx512',['../intel-ipsec-mb_8h.html#a730f18bd7cdcbbb995e24931307867fc',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5fsse_3',['aes_cmac_subkey_gen_sse',['../intel-ipsec-mb_8h.html#a4dafe82f4933670f68e1bf34d55a691d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen2_4',['aes_gcm_dec_128_avx_gen2',['../intel-ipsec-mb_8h.html#a8c9a7e51599dbce89a416e0d8422742b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen4_5',['aes_gcm_dec_128_avx_gen4',['../intel-ipsec-mb_8h.html#ae1e8e1b4425af520048a9e1fe5689e4d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen2_6',['aes_gcm_dec_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#ac173e4fb252a820edb25ad790065ad39',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen4_7',['aes_gcm_dec_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a8e2bc4fff9de3c70ac140ff30e1cdce0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5fsse_8',['aes_gcm_dec_128_finalize_sse',['../intel-ipsec-mb_8h.html#a08603861ac18d1dce7b322b0c3fb69c5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fsse_9',['aes_gcm_dec_128_sse',['../intel-ipsec-mb_8h.html#add2f5aafaf55cd0e0b20b2b62fcb025d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen2_10',['aes_gcm_dec_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#a796e1df6d2aa233a6e28a867c1a2c628',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen4_11',['aes_gcm_dec_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#a4464684ff4cf4e9cb978409ad350c4d6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5fsse_12',['aes_gcm_dec_128_update_sse',['../intel-ipsec-mb_8h.html#a2216e461e916668b70d547513b0cf26f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen2_13',['aes_gcm_dec_192_avx_gen2',['../intel-ipsec-mb_8h.html#ad45e83e263506684c786b4e7012055ac',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen4_14',['aes_gcm_dec_192_avx_gen4',['../intel-ipsec-mb_8h.html#a5ccd87fbcfffc238fbcd1fc816eb06ee',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen2_15',['aes_gcm_dec_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#aacc19f04c9ac80d1bf7391661e191798',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen4_16',['aes_gcm_dec_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#afa136a3c06d45e914d15415dee27f001',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5fsse_17',['aes_gcm_dec_192_finalize_sse',['../intel-ipsec-mb_8h.html#a0fd357a1794d78ac2168bc7eff0442ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fsse_18',['aes_gcm_dec_192_sse',['../intel-ipsec-mb_8h.html#acaff83c8ca568201805e0ada5a79f6e1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen2_19',['aes_gcm_dec_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#aa81615f7ddfc8e1a794c5e278d22f7fc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen4_20',['aes_gcm_dec_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a615c26f03ba08c8cde73cb074bf6b363',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5fsse_21',['aes_gcm_dec_192_update_sse',['../intel-ipsec-mb_8h.html#a58455aec5650ecfc6077c69c97031f63',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen2_22',['aes_gcm_dec_256_avx_gen2',['../intel-ipsec-mb_8h.html#a9b79136dc5996baf598e3aeee520871c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen4_23',['aes_gcm_dec_256_avx_gen4',['../intel-ipsec-mb_8h.html#ac16384799933d4f7b11cb56d9c66f5da',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen2_24',['aes_gcm_dec_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a72f76caa84e2803b2b4a26c81623f89e',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen4_25',['aes_gcm_dec_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a791be274839adc3d26d9a32aa319dc3b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5fsse_26',['aes_gcm_dec_256_finalize_sse',['../intel-ipsec-mb_8h.html#adf7aa2e79dce7a9ed79134c086715fd0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fsse_27',['aes_gcm_dec_256_sse',['../intel-ipsec-mb_8h.html#afd8763c66749e9bc5e4e5eb2ac130e43',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen2_28',['aes_gcm_dec_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a0c31c2e2760edc7b252214214159cd7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen4_29',['aes_gcm_dec_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#a141303c38f4eed1245b542b6e9473296',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5fsse_30',['aes_gcm_dec_256_update_sse',['../intel-ipsec-mb_8h.html#ab1a51ae4b3cf3e8fdb618773d7b24b68',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen2_31',['aes_gcm_enc_128_avx_gen2',['../intel-ipsec-mb_8h.html#af33d7afa861dcedc2c284330a5a66164',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen4_32',['aes_gcm_enc_128_avx_gen4',['../intel-ipsec-mb_8h.html#aa17c919e873b6cfd7ac990977eef7601',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen2_33',['aes_gcm_enc_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a40a83ee6dec0440cd99e5ff600e3803d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen4_34',['aes_gcm_enc_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a004c38b922d25d8631252854426ebf44',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5fsse_35',['aes_gcm_enc_128_finalize_sse',['../intel-ipsec-mb_8h.html#af63ae2246c6d89b7d6630623948d3886',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fsse_36',['aes_gcm_enc_128_sse',['../intel-ipsec-mb_8h.html#aad9e6c466a5274b974d20330f91add50',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen2_37',['aes_gcm_enc_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#ae3459d38a2ee2e1ce04b9f9f4bf55ea5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen4_38',['aes_gcm_enc_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#abb5570a161a90391e25b2016bc98381b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5fsse_39',['aes_gcm_enc_128_update_sse',['../intel-ipsec-mb_8h.html#a7b12cabbfebb420022a19124aff78a3f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen2_40',['aes_gcm_enc_192_avx_gen2',['../intel-ipsec-mb_8h.html#a0ff9569b5072896a4119a4c7d3849bfc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen4_41',['aes_gcm_enc_192_avx_gen4',['../intel-ipsec-mb_8h.html#a9b462a3c00471e4338f0bae7d5abbda7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen2_42',['aes_gcm_enc_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a5177757ae4a75ba41d5e7a8a534edc06',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen4_43',['aes_gcm_enc_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a4f76beaee47ffeeeb6f94a132f898339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5fsse_44',['aes_gcm_enc_192_finalize_sse',['../intel-ipsec-mb_8h.html#a5dcb5c37d1a5377967888e9b76208f8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fsse_45',['aes_gcm_enc_192_sse',['../intel-ipsec-mb_8h.html#adec2f9778622b3e226f12bb326e75076',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen2_46',['aes_gcm_enc_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#a548fcbb89ff5fb798bf71ab8e4ac88eb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen4_47',['aes_gcm_enc_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a174447b05661285b2729805f66bcef81',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5fsse_48',['aes_gcm_enc_192_update_sse',['../intel-ipsec-mb_8h.html#a46c3df131f9e1e3ed1a283349c4072f7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen2_49',['aes_gcm_enc_256_avx_gen2',['../intel-ipsec-mb_8h.html#a45dea093699e6e2dd462b6cafccee672',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen4_50',['aes_gcm_enc_256_avx_gen4',['../intel-ipsec-mb_8h.html#a52c0913f10dd41fea5fe328037561ff8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen2_51',['aes_gcm_enc_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a7fa5d4f44bb44600fa7b86697ccc385f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen4_52',['aes_gcm_enc_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a873a75e2175acd3741b5cfc41180fe2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5fsse_53',['aes_gcm_enc_256_finalize_sse',['../intel-ipsec-mb_8h.html#afcdedeabc7516e7506fa5c81d07f6fd8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fsse_54',['aes_gcm_enc_256_sse',['../intel-ipsec-mb_8h.html#a460ad1ebcb7df39e39c9aca214d8109c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen2_55',['aes_gcm_enc_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a351c2bf07dcab7e01570273c9df1e56f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen4_56',['aes_gcm_enc_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#ad9592051cf26814ea5830826b553c547',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5fsse_57',['aes_gcm_enc_256_update_sse',['../intel-ipsec-mb_8h.html#ab831bdb2193a5adc54b51285f23ad5ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen2_58',['aes_gcm_init_128_avx_gen2',['../intel-ipsec-mb_8h.html#a7a9a24d480dcedac1aa40d53e01b8a76',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen4_59',['aes_gcm_init_128_avx_gen4',['../intel-ipsec-mb_8h.html#a8d225d08a5cf144208dacb155e192cb6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5fsse_60',['aes_gcm_init_128_sse',['../intel-ipsec-mb_8h.html#aca7ab49c370f8f7cb82f93db199bfd1c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen2_61',['aes_gcm_init_192_avx_gen2',['../intel-ipsec-mb_8h.html#ac24548c2955399fd84b03216ad5b36fb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen4_62',['aes_gcm_init_192_avx_gen4',['../intel-ipsec-mb_8h.html#a757cd199a2e993c6610bbcdfc1da1ae4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5fsse_63',['aes_gcm_init_192_sse',['../intel-ipsec-mb_8h.html#a5bbf84135701f572ec2ecb0c0898848b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen2_64',['aes_gcm_init_256_avx_gen2',['../intel-ipsec-mb_8h.html#a3dd74260f4f34f7a53253c63ad3a6701',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen4_65',['aes_gcm_init_256_avx_gen4',['../intel-ipsec-mb_8h.html#a99310b73bc9cde6b69b682e72cd0efc1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5fsse_66',['aes_gcm_init_256_sse',['../intel-ipsec-mb_8h.html#a09949bd5112eec14773dce239ca39033',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen2_67',['aes_gcm_pre_128_avx_gen2',['../intel-ipsec-mb_8h.html#a2f60098fff25e8b71b35e42cc7854339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen4_68',['aes_gcm_pre_128_avx_gen4',['../intel-ipsec-mb_8h.html#a3d7b8bc454b44abbeaa8f435280f8cb1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5fsse_69',['aes_gcm_pre_128_sse',['../intel-ipsec-mb_8h.html#a5d02ac2586befa6cd3fd4921858c33fd',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen2_70',['aes_gcm_pre_192_avx_gen2',['../intel-ipsec-mb_8h.html#a6c9c0b179633aca3ba5c520bee32662f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen4_71',['aes_gcm_pre_192_avx_gen4',['../intel-ipsec-mb_8h.html#ab71c95df9bd059e1a5e108d2513a0e2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5fsse_72',['aes_gcm_pre_192_sse',['../intel-ipsec-mb_8h.html#ac0ebc1e7678bde75c25d5c0ec7ab2f7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen2_73',['aes_gcm_pre_256_avx_gen2',['../intel-ipsec-mb_8h.html#aaaabf4acdd43f98b6c2d31c8013ebbf4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen4_74',['aes_gcm_pre_256_avx_gen4',['../intel-ipsec-mb_8h.html#a082773f106f399ca73afc56583805a8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5fsse_75',['aes_gcm_pre_256_sse',['../intel-ipsec-mb_8h.html#a18fac8ea0f3acabb4d5ffab1152f477f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen2_76',['aes_gcm_precomp_128_avx_gen2',['../intel-ipsec-mb_8h.html#ade16ed4c31baa50f8a10e3c85156a942',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen4_77',['aes_gcm_precomp_128_avx_gen4',['../intel-ipsec-mb_8h.html#aecf7277999128599c6cf8432b242ba36',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5fsse_78',['aes_gcm_precomp_128_sse',['../intel-ipsec-mb_8h.html#a46253857355ddae8d17bab1403289d0f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen2_79',['aes_gcm_precomp_192_avx_gen2',['../intel-ipsec-mb_8h.html#ab0055e3b7d00cdb578647921207a1de7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen4_80',['aes_gcm_precomp_192_avx_gen4',['../intel-ipsec-mb_8h.html#a6cb54cf9321dfb95af2eeb812f41fa8f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5fsse_81',['aes_gcm_precomp_192_sse',['../intel-ipsec-mb_8h.html#a72d93e3249806eee186d465dc04c09e6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen2_82',['aes_gcm_precomp_256_avx_gen2',['../intel-ipsec-mb_8h.html#a2a883085207ee21caca109ab8250db08',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen4_83',['aes_gcm_precomp_256_avx_gen4',['../intel-ipsec-mb_8h.html#abef922c52013630ad70273cef095c564',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5fsse_84',['aes_gcm_precomp_256_sse',['../intel-ipsec-mb_8h.html#a18001897ea80cdce1f4f608b0922f767',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx_85',['aes_keyexp_128_avx',['../intel-ipsec-mb_8h.html#aec7fe23c32ee94de88e5feba8f373a04',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx2_86',['aes_keyexp_128_avx2',['../intel-ipsec-mb_8h.html#a4fc7f88bdb263ae0d45f37af7e4cb930',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx512_87',['aes_keyexp_128_avx512',['../intel-ipsec-mb_8h.html#a9b7faaf3e5235d0a3c4b784f1035896d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx_88',['aes_keyexp_128_enc_avx',['../intel-ipsec-mb_8h.html#ac7d7c889a25730d82ddee27bad7f0a63',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx2_89',['aes_keyexp_128_enc_avx2',['../intel-ipsec-mb_8h.html#a84e76bdfb5a69dba7b515dd0b4fb4312',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx512_90',['aes_keyexp_128_enc_avx512',['../intel-ipsec-mb_8h.html#a68e6a0e99a2091d3630bf3315fe9d95c',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5fsse_91',['aes_keyexp_128_enc_sse',['../intel-ipsec-mb_8h.html#a270621a2f73134d8d021b5dd57ad6817',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fsse_92',['aes_keyexp_128_sse',['../intel-ipsec-mb_8h.html#a32bcda3484b33b7d01124dbfeb9a229b',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx_93',['aes_keyexp_192_avx',['../intel-ipsec-mb_8h.html#a3bc56101b48217094086ae3120b0e086',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx2_94',['aes_keyexp_192_avx2',['../intel-ipsec-mb_8h.html#aba6dd2e633819829740cb7917ba6b1b2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx512_95',['aes_keyexp_192_avx512',['../intel-ipsec-mb_8h.html#a1f69c0a43de83f8fba19107a63c07831',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx_96',['aes_keyexp_192_enc_avx',['../intel-ipsec-mb_8h.html#a5350b81ab574c990d6a5b9dc3b203c51',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx2_97',['aes_keyexp_192_enc_avx2',['../intel-ipsec-mb_8h.html#a460853c072adc3a0f431e861d3e5bba1',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx512_98',['aes_keyexp_192_enc_avx512',['../intel-ipsec-mb_8h.html#a72807e8300208b22b5d532e274385127',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5fsse_99',['aes_keyexp_192_enc_sse',['../intel-ipsec-mb_8h.html#a0a843e722f2af7a645340f6a98cfa4c2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fsse_100',['aes_keyexp_192_sse',['../intel-ipsec-mb_8h.html#a7681d777bb5f991ab59b5581bc854562',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx_101',['aes_keyexp_256_avx',['../intel-ipsec-mb_8h.html#a5b92ee3873b2e2cd02c9312b6040ee91',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx2_102',['aes_keyexp_256_avx2',['../intel-ipsec-mb_8h.html#af1f440855a00edbf95c124a1e1d18c37',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx512_103',['aes_keyexp_256_avx512',['../intel-ipsec-mb_8h.html#a37e0f61656618413fcfd919dfe79ca45',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx_104',['aes_keyexp_256_enc_avx',['../intel-ipsec-mb_8h.html#a22096fdd9b5d32095571f04a838d66a9',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx2_105',['aes_keyexp_256_enc_avx2',['../intel-ipsec-mb_8h.html#a73da5625112ad845a3ec94ee84590f54',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx512_106',['aes_keyexp_256_enc_avx512',['../intel-ipsec-mb_8h.html#a9fb47323b270bffc87ad8e77985e827d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5fsse_107',['aes_keyexp_256_enc_sse',['../intel-ipsec-mb_8h.html#afa369dee8def2ef620e7f33db86a3356',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fsse_108',['aes_keyexp_256_sse',['../intel-ipsec-mb_8h.html#a5105a8d995cd8e4354c7de7ac94886cb',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx_109',['aes_xcbc_expand_key_avx',['../intel-ipsec-mb_8h.html#afe460e4ff6a7b0c7a6edcc87e9976121',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx2_110',['aes_xcbc_expand_key_avx2',['../intel-ipsec-mb_8h.html#ac3f27672ce30d74f5cdc28f1903f68b1',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx512_111',['aes_xcbc_expand_key_avx512',['../intel-ipsec-mb_8h.html#a1308be648182642b8d73ff27d9344564',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5fsse_112',['aes_xcbc_expand_key_sse',['../intel-ipsec-mb_8h.html#aefb28e46620cbf65a2af3073b2401e40',1,'intel-ipsec-mb.h']]], + ['alloc_5fmb_5fmgr_113',['alloc_mb_mgr',['../intel-ipsec-mb_8h.html#ab8dc617feb010b0f4cd74fd9e1eabb5e',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_1.js b/docs/search/functions_1.js new file mode 100644 index 0000000000000000000000000000000000000000..bae33a8e5e58e1d9bb862baf7c7d0f97200a4897 --- /dev/null +++ b/docs/search/functions_1.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['des_5fcfb_5fone_0',['des_cfb_one',['../intel-ipsec-mb_8h.html#a71a0353411f5307a2177ae9b2814f522',1,'intel-ipsec-mb.h']]], + ['des_5fkey_5fschedule_1',['des_key_schedule',['../intel-ipsec-mb_8h.html#a9911462fe4689c7420b827bb13fc3440',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_4.js b/docs/search/functions_4.js new file mode 100644 index 0000000000000000000000000000000000000000..28271cf26558011ad0565ee7e8e96a20b88759e2 --- /dev/null +++ b/docs/search/functions_4.js @@ -0,0 +1,20 @@ +var searchData= +[ + ['imb_5fclear_5fmem_0',['imb_clear_mem',['../intel-ipsec-mb_8h.html#aecdd69541b6d2a83d19ca804ef4b78d6',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ferrno_1',['imb_get_errno',['../intel-ipsec-mb_8h.html#a79f60b2bffbe91dac3e4f8b486e7ab10',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ffeature_5fflags_2',['imb_get_feature_flags',['../intel-ipsec-mb_8h.html#a725703ff6da6b402bde7041a2fabfddf',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fmb_5fmgr_5fsize_3',['imb_get_mb_mgr_size',['../intel-ipsec-mb_8h.html#ac8379ada0d6ec030ebad9828fbea9459',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fstrerror_4',['imb_get_strerror',['../intel-ipsec-mb_8h.html#a61f664448a862fbb02e728ce518ff8e4',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_5',['imb_get_version',['../intel-ipsec-mb_8h.html#aee02bcb3a5e5eaee3632086f75c3d78d',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_5fstr_6',['imb_get_version_str',['../intel-ipsec-mb_8h.html#a4490e7b324236315f1a9e141b39f99f2',1,'intel-ipsec-mb.h']]], + ['imb_5fhmac_5fipad_5fopad_7',['imb_hmac_ipad_opad',['../intel-ipsec-mb_8h.html#ab5ecb9fd270d0bfc42e46c5a3a9225fe',1,'intel-ipsec-mb.h']]], + ['imb_5fquic_5faes_5fgcm_8',['imb_quic_aes_gcm',['../intel-ipsec-mb_8h.html#a8faae8f9387983904125d862c4cac9ab',1,'intel-ipsec-mb.h']]], + ['imb_5fquic_5fhp_5faes_5fecb_9',['imb_quic_hp_aes_ecb',['../intel-ipsec-mb_8h.html#a89f9807d10f3459444bb649caf9b64ca',1,'intel-ipsec-mb.h']]], + ['imb_5fset_5fpointers_5fmb_5fmgr_10',['imb_set_pointers_mb_mgr',['../intel-ipsec-mb_8h.html#a3a825e903d1f8f8611b98b60fcaa5a81',1,'intel-ipsec-mb.h']]], + ['imb_5fset_5fsession_11',['imb_set_session',['../intel-ipsec-mb_8h.html#ada304337e78ef5a462c97c1f8151ef67',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5fauto_12',['init_mb_mgr_auto',['../intel-ipsec-mb_8h.html#a2cbbd032835837fd0aa83b04f5337638',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx_13',['init_mb_mgr_avx',['../intel-ipsec-mb_8h.html#aee547b1b9e70e12c84d9949317e41c6c',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx2_14',['init_mb_mgr_avx2',['../intel-ipsec-mb_8h.html#a71b6563b6b4c0c504cd9b8302a242cfa',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx512_15',['init_mb_mgr_avx512',['../intel-ipsec-mb_8h.html#a98932a3c61d798b0c8a560dcc3dce16f',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5fsse_16',['init_mb_mgr_sse',['../intel-ipsec-mb_8h.html#a2ad1941fe5d77b8bb2d6445679e3b0c5',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_8.js b/docs/search/functions_8.js new file mode 100644 index 0000000000000000000000000000000000000000..e464b1e291dbc06608377bca47dba51977402aab --- /dev/null +++ b/docs/search/functions_8.js @@ -0,0 +1,13 @@ +var searchData= +[ + ['snow3g_5ff8_5fiv_5fgen_0',['snow3g_f8_iv_gen',['../intel-ipsec-mb_8h.html#a2cbdb64c0b305cb5af010f9c98f70a43',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff9_5fiv_5fgen_1',['snow3g_f9_iv_gen',['../intel-ipsec-mb_8h.html#a93e7a99927bc37fb4cb03f966fa4ebdd',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx_2',['submit_job_avx',['../intel-ipsec-mb_8h.html#a950bf3b7a3c63a371b561ea20aaf3a18',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx2_3',['submit_job_avx2',['../intel-ipsec-mb_8h.html#a8913995859e45005f8de57a3456c93cf',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx512_4',['submit_job_avx512',['../intel-ipsec-mb_8h.html#a75cc0d090470e47cd8be04451ee31a68',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx_5',['submit_job_nocheck_avx',['../intel-ipsec-mb_8h.html#ac7b339426753d3a7c44625a5ded3170a',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx2_6',['submit_job_nocheck_avx2',['../intel-ipsec-mb_8h.html#a72caaf8162ae6f802f7c1fe570afe42e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx512_7',['submit_job_nocheck_avx512',['../intel-ipsec-mb_8h.html#a5a95b9268508d68b9027df98167d22cd',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5fsse_8',['submit_job_nocheck_sse',['../intel-ipsec-mb_8h.html#a35e4c8a3d2eba8a612f021572621f98e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fsse_9',['submit_job_sse',['../intel-ipsec-mb_8h.html#a7700e90058a74980bfd7d462a19d32a0',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/mag.svg b/docs/search/mag.svg new file mode 100644 index 0000000000000000000000000000000000000000..ffb6cf0d0251cb0eb2b0173fbd77ee2373385527 --- /dev/null +++ b/docs/search/mag.svg @@ -0,0 +1,24 @@ + + + + + + + diff --git a/docs/search/mag_d.svg b/docs/search/mag_d.svg new file mode 100644 index 0000000000000000000000000000000000000000..4122773f92c32ce6bca14d8bf407c11c635bfcf3 --- /dev/null +++ b/docs/search/mag_d.svg @@ -0,0 +1,24 @@ + + + + + + + diff --git a/docs/search/mag_sel.svg b/docs/search/mag_sel.svg new file mode 100644 index 0000000000000000000000000000000000000000..553dba8773264b110b8c04cfaecf8d94df01a975 --- /dev/null +++ b/docs/search/mag_sel.svg @@ -0,0 +1,31 @@ + + + + + + + + + diff --git a/docs/search/mag_seld.svg b/docs/search/mag_seld.svg new file mode 100644 index 0000000000000000000000000000000000000000..c906f84c83a39ee10705c40b722d4eda15966327 --- /dev/null +++ b/docs/search/mag_seld.svg @@ -0,0 +1,31 @@ + + + + + + + + + diff --git a/docs/search/search.css b/docs/search/search.css new file mode 100644 index 0000000000000000000000000000000000000000..a53214fcdc2d035878a0b53203327822be3dd424 --- /dev/null +++ b/docs/search/search.css @@ -0,0 +1,286 @@ +/*---------------- Search Box */ + +#MSearchBox { + position: absolute; + right: 5px; +} +/*---------------- Search box styling */ + +.SRPage * { + font-weight: normal; + line-height: normal; +} + +dark-mode-toggle { + margin-left: 5px; + display: flex; + float: right; +} + +#MSearchBox { + display: inline-block; + white-space : nowrap; + background: var(--search-background-color); + border-radius: 0.65em; + box-shadow: var(--search-box-shadow); + z-index: 102; +} + +#MSearchBox .left { + display: inline-block; + vertical-align: middle; + height: 1.4em; +} + +#MSearchSelect { + display: inline-block; + vertical-align: middle; + width: 20px; + height: 19px; + background-image: var(--search-magnification-select-image); + margin: 0 0 0 0.3em; + padding: 0; +} + +#MSearchSelectExt { + display: inline-block; + vertical-align: middle; + width: 10px; + height: 19px; + background-image: var(--search-magnification-image); + margin: 0 0 0 0.5em; + padding: 0; +} + + +#MSearchField { + display: inline-block; + vertical-align: middle; + width: 7.5em; + height: 19px; + margin: 0 0.15em; + padding: 0; + line-height: 1em; + border:none; + color: var(--search-foreground-color); + outline: none; + font-family: var(--font-family-search); + -webkit-border-radius: 0px; + border-radius: 0px; + background: none; +} + +@media(hover: none) { + /* to avoid zooming on iOS */ + #MSearchField { + font-size: 16px; + } +} + +#MSearchBox .right { + display: inline-block; + vertical-align: middle; + width: 1.4em; + height: 1.4em; +} + +#MSearchClose { + display: none; + font-size: inherit; + background : none; + border: none; + margin: 0; + padding: 0; + outline: none; + +} + +#MSearchCloseImg { + padding: 0.3em; + margin: 0; +} + +.MSearchBoxActive #MSearchField { + color: var(--search-active-color); +} + + + +/*---------------- Search filter selection */ + +#MSearchSelectWindow { + display: none; + position: absolute; + left: 0; top: 0; + border: 1px solid var(--search-filter-border-color); + background-color: var(--search-filter-background-color); + z-index: 10001; + padding-top: 4px; + padding-bottom: 4px; + -moz-border-radius: 4px; + -webkit-border-top-left-radius: 4px; + -webkit-border-top-right-radius: 4px; + -webkit-border-bottom-left-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); +} + +.SelectItem { + font: 8pt var(--font-family-search); + padding-left: 2px; + padding-right: 12px; + border: 0px; +} + +span.SelectionMark { + margin-right: 4px; + font-family: var(--font-family-monospace); + outline-style: none; + text-decoration: none; +} + +a.SelectItem { + display: block; + outline-style: none; + color: var(--search-filter-foreground-color); + text-decoration: none; + padding-left: 6px; + padding-right: 12px; +} + +a.SelectItem:focus, +a.SelectItem:active { + color: var(--search-filter-foreground-color); + outline-style: none; + text-decoration: none; +} + +a.SelectItem:hover { + color: var(--search-filter-highlight-text-color); + background-color: var(--search-filter-highlight-bg-color); + outline-style: none; + text-decoration: none; + cursor: pointer; + display: block; +} + +/*---------------- Search results window */ + +iframe#MSearchResults { + /*width: 60ex;*/ + height: 15em; +} + +#MSearchResultsWindow { + display: none; + position: absolute; + left: 0; top: 0; + border: 1px solid var(--search-results-border-color); + background-color: var(--search-results-background-color); + z-index:10000; + width: 300px; + height: 400px; + overflow: auto; +} + +/* ----------------------------------- */ + + +#SRIndex { + clear:both; +} + +.SREntry { + font-size: 10pt; + padding-left: 1ex; +} + +.SRPage .SREntry { + font-size: 8pt; + padding: 1px 5px; +} + +div.SRPage { + margin: 5px 2px; + background-color: var(--search-results-background-color); +} + +.SRChildren { + padding-left: 3ex; padding-bottom: .5em +} + +.SRPage .SRChildren { + display: none; +} + +.SRSymbol { + font-weight: bold; + color: var(--search-results-foreground-color); + font-family: var(--font-family-search); + text-decoration: none; + outline: none; +} + +a.SRScope { + display: block; + color: var(--search-results-foreground-color); + font-family: var(--font-family-search); + font-size: 8pt; + text-decoration: none; + outline: none; +} + +a.SRSymbol:focus, a.SRSymbol:active, +a.SRScope:focus, a.SRScope:active { + text-decoration: underline; +} + +span.SRScope { + padding-left: 4px; + font-family: var(--font-family-search); +} + +.SRPage .SRStatus { + padding: 2px 5px; + font-size: 8pt; + font-style: italic; + font-family: var(--font-family-search); +} + +.SRResult { + display: none; +} + +div.searchresults { + margin-left: 10px; + margin-right: 10px; +} + +/*---------------- External search page results */ + +.pages b { + color: white; + padding: 5px 5px 3px 5px; + background-image: var(--nav-gradient-active-image-parent); + background-repeat: repeat-x; + text-shadow: 0 1px 1px #000000; +} + +.pages { + line-height: 17px; + margin-left: 4px; + text-decoration: none; +} + +.hl { + font-weight: bold; +} + +#searchresults { + margin-bottom: 20px; +} + +.searchpages { + margin-top: 10px; +} + diff --git a/docs/search/search.js b/docs/search/search.js new file mode 100644 index 0000000000000000000000000000000000000000..9b7a52a195cab952f3aad45b2cbc64f251b3a4cb --- /dev/null +++ b/docs/search/search.js @@ -0,0 +1,820 @@ +/* + @licstart The following is the entire license notice for the JavaScript code in this file. + + The MIT License (MIT) + + Copyright (C) 1997-2020 by Dimitri van Heesch + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + @licend The above is the entire license notice for the JavaScript code in this file + */ +function convertToId(search) +{ + var result = ''; + for (i=0;i do a search + { + this.Search(); + } + } + + this.OnSearchSelectKey = function(evt) + { + var e = (evt) ? evt : window.event; // for IE + if (e.keyCode==40 && this.searchIndex0) // Up + { + this.searchIndex--; + this.OnSelectItem(this.searchIndex); + } + else if (e.keyCode==13 || e.keyCode==27) + { + this.OnSelectItem(this.searchIndex); + this.CloseSelectionWindow(); + this.DOMSearchField().focus(); + } + return false; + } + + // --------- Actions + + // Closes the results window. + this.CloseResultsWindow = function() + { + this.DOMPopupSearchResultsWindow().style.display = 'none'; + this.DOMSearchClose().style.display = 'none'; + this.Activate(false); + } + + this.CloseSelectionWindow = function() + { + this.DOMSearchSelectWindow().style.display = 'none'; + } + + // Performs a search. + this.Search = function() + { + this.keyTimeout = 0; + + // strip leading whitespace + var searchValue = this.DOMSearchField().value.replace(/^ +/, ""); + + var code = searchValue.toLowerCase().charCodeAt(0); + var idxChar = searchValue.substr(0, 1).toLowerCase(); + if ( 0xD800 <= code && code <= 0xDBFF && searchValue > 1) // surrogate pair + { + idxChar = searchValue.substr(0, 2); + } + + var jsFile; + + var idx = indexSectionsWithContent[this.searchIndex].indexOf(idxChar); + if (idx!=-1) + { + var hexCode=idx.toString(16); + jsFile = this.resultsPath + indexSectionNames[this.searchIndex] + '_' + hexCode + '.js'; + } + + var loadJS = function(url, impl, loc){ + var scriptTag = document.createElement('script'); + scriptTag.src = url; + scriptTag.onload = impl; + scriptTag.onreadystatechange = impl; + loc.appendChild(scriptTag); + } + + var domPopupSearchResultsWindow = this.DOMPopupSearchResultsWindow(); + var domSearchBox = this.DOMSearchBox(); + var domPopupSearchResults = this.DOMPopupSearchResults(); + var domSearchClose = this.DOMSearchClose(); + var resultsPath = this.resultsPath; + + var handleResults = function() { + document.getElementById("Loading").style.display="none"; + if (typeof searchData !== 'undefined') { + createResults(resultsPath); + document.getElementById("NoMatches").style.display="none"; + } + + if (idx!=-1) { + searchResults.Search(searchValue); + } else { // no file with search results => force empty search results + searchResults.Search('===='); + } + + if (domPopupSearchResultsWindow.style.display!='block') + { + domSearchClose.style.display = 'inline-block'; + var left = getXPos(domSearchBox) + 150; + var top = getYPos(domSearchBox) + 20; + domPopupSearchResultsWindow.style.display = 'block'; + left -= domPopupSearchResults.offsetWidth; + var maxWidth = document.body.clientWidth; + var maxHeight = document.body.clientHeight; + var width = 300; + if (left<10) left=10; + if (width+left+8>maxWidth) width=maxWidth-left-8; + var height = 400; + if (height+top+8>maxHeight) height=maxHeight-top-8; + domPopupSearchResultsWindow.style.top = top + 'px'; + domPopupSearchResultsWindow.style.left = left + 'px'; + domPopupSearchResultsWindow.style.width = width + 'px'; + domPopupSearchResultsWindow.style.height = height + 'px'; + } + } + + if (jsFile) { + loadJS(jsFile, handleResults, this.DOMPopupSearchResultsWindow()); + } else { + handleResults(); + } + + this.lastSearchValue = searchValue; + } + + // -------- Activation Functions + + // Activates or deactivates the search panel, resetting things to + // their default values if necessary. + this.Activate = function(isActive) + { + if (isActive || // open it + this.DOMPopupSearchResultsWindow().style.display == 'block' + ) + { + this.DOMSearchBox().className = 'MSearchBoxActive'; + this.searchActive = true; + } + else if (!isActive) // directly remove the panel + { + this.DOMSearchBox().className = 'MSearchBoxInactive'; + this.searchActive = false; + this.lastSearchValue = '' + this.lastResultsPage = ''; + this.DOMSearchField().value = ''; + } + } +} + +// ----------------------------------------------------------------------- + +// The class that handles everything on the search results page. +function SearchResults(name) +{ + // The number of matches from the last run of . + this.lastMatchCount = 0; + this.lastKey = 0; + this.repeatOn = false; + + // Toggles the visibility of the passed element ID. + this.FindChildElement = function(id) + { + var parentElement = document.getElementById(id); + var element = parentElement.firstChild; + + while (element && element!=parentElement) + { + if (element.nodeName.toLowerCase() == 'div' && element.className == 'SRChildren') + { + return element; + } + + if (element.nodeName.toLowerCase() == 'div' && element.hasChildNodes()) + { + element = element.firstChild; + } + else if (element.nextSibling) + { + element = element.nextSibling; + } + else + { + do + { + element = element.parentNode; + } + while (element && element!=parentElement && !element.nextSibling); + + if (element && element!=parentElement) + { + element = element.nextSibling; + } + } + } + } + + this.Toggle = function(id) + { + var element = this.FindChildElement(id); + if (element) + { + if (element.style.display == 'block') + { + element.style.display = 'none'; + } + else + { + element.style.display = 'block'; + } + } + } + + // Searches for the passed string. If there is no parameter, + // it takes it from the URL query. + // + // Always returns true, since other documents may try to call it + // and that may or may not be possible. + this.Search = function(search) + { + if (!search) // get search word from URL + { + search = window.location.search; + search = search.substring(1); // Remove the leading '?' + search = unescape(search); + } + + search = search.replace(/^ +/, ""); // strip leading spaces + search = search.replace(/ +$/, ""); // strip trailing spaces + search = search.toLowerCase(); + search = convertToId(search); + + var resultRows = document.getElementsByTagName("div"); + var matches = 0; + + var i = 0; + while (i < resultRows.length) + { + var row = resultRows.item(i); + if (row.className == "SRResult") + { + var rowMatchName = row.id.toLowerCase(); + rowMatchName = rowMatchName.replace(/^sr\d*_/, ''); // strip 'sr123_' + + if (search.length<=rowMatchName.length && + rowMatchName.substr(0, search.length)==search) + { + row.style.display = 'block'; + matches++; + } + else + { + row.style.display = 'none'; + } + } + i++; + } + document.getElementById("Searching").style.display='none'; + if (matches == 0) // no results + { + document.getElementById("NoMatches").style.display='block'; + } + else // at least one result + { + document.getElementById("NoMatches").style.display='none'; + } + this.lastMatchCount = matches; + return true; + } + + // return the first item with index index or higher that is visible + this.NavNext = function(index) + { + var focusItem; + while (1) + { + var focusName = 'Item'+index; + focusItem = document.getElementById(focusName); + if (focusItem && focusItem.parentNode.parentNode.style.display=='block') + { + break; + } + else if (!focusItem) // last element + { + break; + } + focusItem=null; + index++; + } + return focusItem; + } + + this.NavPrev = function(index) + { + var focusItem; + while (1) + { + var focusName = 'Item'+index; + focusItem = document.getElementById(focusName); + if (focusItem && focusItem.parentNode.parentNode.style.display=='block') + { + break; + } + else if (!focusItem) // last element + { + break; + } + focusItem=null; + index--; + } + return focusItem; + } + + this.ProcessKeys = function(e) + { + if (e.type == "keydown") + { + this.repeatOn = false; + this.lastKey = e.keyCode; + } + else if (e.type == "keypress") + { + if (!this.repeatOn) + { + if (this.lastKey) this.repeatOn = true; + return false; // ignore first keypress after keydown + } + } + else if (e.type == "keyup") + { + this.lastKey = 0; + this.repeatOn = false; + } + return this.lastKey!=0; + } + + this.Nav = function(evt,itemIndex) + { + var e = (evt) ? evt : window.event; // for IE + if (e.keyCode==13) return true; + if (!this.ProcessKeys(e)) return false; + + if (this.lastKey==38) // Up + { + var newIndex = itemIndex-1; + var focusItem = this.NavPrev(newIndex); + if (focusItem) + { + var child = this.FindChildElement(focusItem.parentNode.parentNode.id); + if (child && child.style.display == 'block') // children visible + { + var n=0; + var tmpElem; + while (1) // search for last child + { + tmpElem = document.getElementById('Item'+newIndex+'_c'+n); + if (tmpElem) + { + focusItem = tmpElem; + } + else // found it! + { + break; + } + n++; + } + } + } + if (focusItem) + { + focusItem.focus(); + } + else // return focus to search field + { + document.getElementById("MSearchField").focus(); + } + } + else if (this.lastKey==40) // Down + { + var newIndex = itemIndex+1; + var focusItem; + var item = document.getElementById('Item'+itemIndex); + var elem = this.FindChildElement(item.parentNode.parentNode.id); + if (elem && elem.style.display == 'block') // children visible + { + focusItem = document.getElementById('Item'+itemIndex+'_c0'); + } + if (!focusItem) focusItem = this.NavNext(newIndex); + if (focusItem) focusItem.focus(); + } + else if (this.lastKey==39) // Right + { + var item = document.getElementById('Item'+itemIndex); + var elem = this.FindChildElement(item.parentNode.parentNode.id); + if (elem) elem.style.display = 'block'; + } + else if (this.lastKey==37) // Left + { + var item = document.getElementById('Item'+itemIndex); + var elem = this.FindChildElement(item.parentNode.parentNode.id); + if (elem) elem.style.display = 'none'; + } + else if (this.lastKey==27) // Escape + { + searchBox.CloseResultsWindow(); + document.getElementById("MSearchField").focus(); + } + else if (this.lastKey==13) // Enter + { + return true; + } + return false; + } + + this.NavChild = function(evt,itemIndex,childIndex) + { + var e = (evt) ? evt : window.event; // for IE + if (e.keyCode==13) return true; + if (!this.ProcessKeys(e)) return false; + + if (this.lastKey==38) // Up + { + if (childIndex>0) + { + var newIndex = childIndex-1; + document.getElementById('Item'+itemIndex+'_c'+newIndex).focus(); + } + else // already at first child, jump to parent + { + document.getElementById('Item'+itemIndex).focus(); + } + } + else if (this.lastKey==40) // Down + { + var newIndex = childIndex+1; + var elem = document.getElementById('Item'+itemIndex+'_c'+newIndex); + if (!elem) // last child, jump to parent next parent + { + elem = this.NavNext(itemIndex+1); + } + if (elem) + { + elem.focus(); + } + } + else if (this.lastKey==27) // Escape + { + searchBox.CloseResultsWindow(); + document.getElementById("MSearchField").focus(); + } + else if (this.lastKey==13) // Enter + { + return true; + } + return false; + } +} + +function setKeyActions(elem,action) +{ + elem.setAttribute('onkeydown',action); + elem.setAttribute('onkeypress',action); + elem.setAttribute('onkeyup',action); +} + +function setClassAttr(elem,attr) +{ + elem.setAttribute('class',attr); + elem.setAttribute('className',attr); +} + +function createResults(resultsPath) +{ + var results = document.getElementById("SRResults"); + results.innerHTML = ''; + for (var e=0; e + + + + + + +intel-ipsec-mb: IMB_JOB Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
IMB_JOB Struct Reference
+
+
+ +

#include <intel-ipsec-mb.h>

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Data Fields

const void * enc_keys
 
const void * dec_keys
 
uint64_t key_len_in_bytes
 
union { 
 
   const uint8_t *   src 
 
   const struct IMB_SGL_IOV *   sgl_io_segs 
 
};  
 
union { 
 
   uint8_t *   dst 
 
   uint64_t   num_sgl_io_segs 
 
};  
 
union { 
 
   uint64_t   cipher_start_src_offset_in_bytes 
 
   uint64_t   cipher_start_src_offset_in_bits 
 
   uint64_t   cipher_start_offset_in_bits 
 
};  
 
union { 
 
   uint64_t   msg_len_to_cipher_in_bytes 
 
   uint64_t   msg_len_to_cipher_in_bits 
 
};  
 
uint64_t hash_start_src_offset_in_bytes
 
union { 
 
   uint64_t   msg_len_to_hash_in_bytes 
 
   uint64_t   msg_len_to_hash_in_bits 
 
};  
 
const uint8_t * iv
 
uint64_t iv_len_in_bytes
 
uint8_t * auth_tag_output
 
uint64_t auth_tag_output_len_in_bytes
 
union { 
 
   struct _HMAC_specific_fields { 
 
      const uint8_t *   _hashed_auth_key_xor_ipad 
 
      const uint8_t *   _hashed_auth_key_xor_opad 
 
   }   HMAC 
 
   struct _AES_XCBC_specific_fields { 
 
      const uint32_t *   _k1_expanded 
 
      const uint8_t *   _k2 
 
      const uint8_t *   _k3 
 
   }   XCBC 
 
   struct _AES_CCM_specific_fields { 
 
      const void *   aad 
 
      uint64_t   aad_len_in_bytes 
 
   }   CCM 
 
   struct _AES_CMAC_specific_fields { 
 
      const void *   _key_expanded 
 
      const void *   _skey1 
 
      const void *   _skey2 
 
   }   CMAC 
 
   struct _AES_GCM_specific_fields { 
 
      const void *   aad 
 
      uint64_t   aad_len_in_bytes 
 
      struct gcm_context_data *   ctx 
 
   }   GCM 
 
   struct _ZUC_EIA3_specific_fields { 
 
      const uint8_t *   _key 
 
      const uint8_t *   _iv 
 
      const uint8_t *   _iv23 
 
   }   ZUC_EIA3 
 
   struct _SNOW3G_UIA2_specific_fields { 
 
      const void *   _key 
 
      const void *   _iv 
 
   }   SNOW3G_UIA2 
 
   struct _KASUMI_UIA1_specific_fields { 
 
      const void *   _key 
 
   }   KASUMI_UIA1 
 
   struct _AES_GMAC_specific_fields { 
 
      const struct gcm_key_data *   _key 
 
      const void *   _iv 
 
      uint64_t   iv_len_in_bytes 
 
   }   GMAC 
 
   struct _GHASH_specific_fields { 
 
      const struct gcm_key_data *   _key 
 
      const void *   _init_tag 
 
   }   GHASH 
 
   struct _POLY1305_specific_fields { 
 
      const void *   _key 
 
   }   POLY1305 
 
   struct _CHACHA20_POLY1305_specific_fields { 
 
      const void *   aad 
 
      uint64_t   aad_len_in_bytes 
 
      struct chacha20_poly1305_context_data *   ctx 
 
   }   CHACHA20_POLY1305 
 
   struct _SNOW_V_AEAD_specific_fields { 
 
      const void *   aad 
 
      uint64_t   aad_len_in_bytes 
 
      void *   reserved 
 
   }   SNOW_V_AEAD 
 
} u 
 
IMB_STATUS status
 
IMB_CIPHER_MODE cipher_mode
 
IMB_CIPHER_DIRECTION cipher_direction
 
IMB_HASH_ALG hash_alg
 
IMB_CHAIN_ORDER chain_order
 
void * user_data
 
void * user_data2
 
int(* cipher_func )(struct IMB_JOB *)
 
int(* hash_func )(struct IMB_JOB *)
 
IMB_SGL_STATE sgl_state
 
union { 
 
   struct _CBCS_specific_fields { 
 
      void *   next_iv 
 
   }   CBCS 
 
} cipher_fields 
 
void * suite_id [4]
 
uint32_t session_id
 
+

Detailed Description

+

Job structure.

+

For AES, enc_keys and dec_keys are expected to point to expanded keys structure.

    +
  • AES-CTR, AES-ECB and AES-CCM, only enc_keys is used
    • +
  • DOCSIS (AES-CBC + AES-CFB), both pointers are used enc_keys has to be set always for the partial block
    • +
+

For DES, enc_keys and dec_keys are expected to point to DES key schedule.

    +
  • same key schedule used for enc and dec operations
    • +
+

For 3DES, enc_keys and dec_keys are expected to point to an array of 3 pointers for the corresponding 3 key schedules.

    +
  • same key schedule used for enc and dec operations
    • +
+

Field Documentation

+ +

◆ [union]

+ +
+
+ + + + +
union { ... } IMB_JOB
+
+ +
+
+ +

◆ [union]

+ +
+
+ + + + +
union { ... } IMB_JOB
+
+ +
+
+ +

◆ [union]

+ +
+
+ + + + +
union { ... } IMB_JOB
+
+

Offset into input buffer to start ciphering

+ +
+
+ +

◆ [union]

+ +
+
+ + + + +
union { ... } IMB_JOB
+
+

Length of message to cipher

+ +
+
+ +

◆ [union]

+ +
+
+ + + + +
union { ... } IMB_JOB
+
+

Length of message to hash

+ +
+
+ +

◆ _hashed_auth_key_xor_ipad

+ +
+
+ + + + +
const uint8_t* IMB_JOB::_hashed_auth_key_xor_ipad
+
+

Hashed result of HMAC key xor'd with ipad (0x36).

+ +
+
+ +

◆ _hashed_auth_key_xor_opad

+ +
+
+ + + + +
const uint8_t* IMB_JOB::_hashed_auth_key_xor_opad
+
+

Hashed result of HMAC key xor'd with opad (0x5c).

+ +
+
+ +

◆ _init_tag

+ +
+
+ + + + +
const void* IMB_JOB::_init_tag
+
+

initial tag value

+ +
+
+ +

◆ _iv [1/2]

+ +
+
+ + + + +
const uint8_t* IMB_JOB::_iv
+
+

Authentication 25-byte IV (16-byte aligned)

+ +
+
+ +

◆ _iv [2/2]

+ +
+
+ + + + +
const void* IMB_JOB::_iv
+
+

Authentication IV (16-byte aligned)

+

Authentication IV

+ +
+
+ +

◆ _iv23

+ +
+
+ + + + +
const uint8_t* IMB_JOB::_iv23
+
+

Authentication 23-byte IV (16-byte aligned)

+ +
+
+ +

◆ _k1_expanded

+ +
+
+ + + + +
const uint32_t* IMB_JOB::_k1_expanded
+
+

k1 expanded key pointer (16-byte aligned)

+ +
+
+ +

◆ _k2

+ +
+
+ + + + +
const uint8_t* IMB_JOB::_k2
+
+

k2 expanded key pointer (16-byte aligned)

+ +
+
+ +

◆ _k3

+ +
+
+ + + + +
const uint8_t* IMB_JOB::_k3
+
+

k3 expanded key pointer (16-byte aligned)

+ +
+
+ +

◆ _key [1/3]

+ +
+
+ + + + +
const uint8_t* IMB_JOB::_key
+
+

Authentication key (16-byte aligned)

+ +
+
+ +

◆ _key [2/3]

+ +
+
+ + + + +
const void* IMB_JOB::_key
+
+

Authentication key (16-byte aligned)

+

Poly1305 key

+ +
+
+ +

◆ _key [3/3]

+ +
+
+ + + + +
const struct gcm_key_data* IMB_JOB::_key
+
+

Authentication key

+

Expanded GHASH key

+ +
+
+ +

◆ _key_expanded

+ +
+
+ + + + +
const void* IMB_JOB::_key_expanded
+
+

Expanded key (16-byte aligned)

+ +
+
+ +

◆ _skey1

+ +
+
+ + + + +
const void* IMB_JOB::_skey1
+
+

S key 1 (16-byte aligned)

+ +
+
+ +

◆ _skey2

+ +
+
+ + + + +
const void* IMB_JOB::_skey2
+
+

S key 2 (16-byte aligned)

+ +
+
+ +

◆ aad

+ +
+
+ + + + +
const void* IMB_JOB::aad
+
+

Additional Authentication Data (AAD)

+ +
+
+ +

◆ aad_len_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::aad_len_in_bytes
+
+

Length of AAD

+ +
+
+ +

◆ auth_tag_output

+ +
+
+ + + + +
uint8_t* IMB_JOB::auth_tag_output
+
+

Authentication tag output

+ +
+
+ +

◆ auth_tag_output_len_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::auth_tag_output_len_in_bytes
+
+

Authentication tag output length in bytes

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_CBCS_specific_fields IMB_JOB::CBCS
+
+

CBCS specific fields

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_AES_CCM_specific_fields IMB_JOB::CCM
+
+

AES-CCM specific fields

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_CHACHA20_POLY1305_specific_fields IMB_JOB::CHACHA20_POLY1305
+
+

Chacha20-Poly1305 specific fields

+ +
+
+ +

◆ chain_order

+ +
+
+ + + + +
IMB_CHAIN_ORDER IMB_JOB::chain_order
+
+

Chain order (IMB_ORDER_CIPHER_HASH / IMB_ORDER_HASH_CIPHER).

+ +
+
+ +

◆ cipher_direction

+ +
+
+ + + + +
IMB_CIPHER_DIRECTION IMB_JOB::cipher_direction
+
+

Cipher direction

+ +
+
+ +

◆ [union]

+ +
+
+ + + + +
union { ... } IMB_JOB::cipher_fields
+
+

Cipher algorithm-specific fields

+ +
+
+ +

◆ cipher_func

+ +
+
+ + + + +
int(* IMB_JOB::cipher_func) (struct IMB_JOB *)
+
+

Customer cipher function

+ +
+
+ +

◆ cipher_mode

+ +
+
+ + + + +
IMB_CIPHER_MODE IMB_JOB::cipher_mode
+
+

Cipher mode

+ +
+
+ +

◆ cipher_start_offset_in_bits

+ +
+
+ + + + +
uint64_t IMB_JOB::cipher_start_offset_in_bits
+
+

Offset into input buffer to start ciphering (in bits)

+ +
+
+ +

◆ cipher_start_src_offset_in_bits

+ +
+
+ + + + +
uint64_t IMB_JOB::cipher_start_src_offset_in_bits
+
+

Offset into input buffer to start ciphering (in bits)

+ +
+
+ +

◆ cipher_start_src_offset_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::cipher_start_src_offset_in_bytes
+
+

Offset into input buffer to start ciphering (in bytes)

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_AES_CMAC_specific_fields IMB_JOB::CMAC
+
+

AES-CMAC specific fields

+ +
+
+ +

◆ ctx [1/2]

+ +
+
+ + + + +
struct gcm_context_data* IMB_JOB::ctx
+
+

AES-GCM context (for SGL only)

+ +
+
+ +

◆ ctx [2/2]

+ +
+
+ + + + +
struct chacha20_poly1305_context_data* IMB_JOB::ctx
+
+

Chacha20-Poly1305 context (for SGL only)

+ +
+
+ +

◆ dec_keys

+ +
+
+ + + + +
const void* IMB_JOB::dec_keys
+
+

Decryption key pointer

+ +
+
+ +

◆ dst

+ +
+
+ + + + +
uint8_t* IMB_JOB::dst
+
+

Output buffer. May be ciphertext or plaintext. In-place ciphering allowed, i.e. dst = src.

+ +
+
+ +

◆ enc_keys

+ +
+
+ + + + +
const void* IMB_JOB::enc_keys
+
+

Encryption key pointer

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_AES_GCM_specific_fields IMB_JOB::GCM
+
+

AES-GCM specific fields

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_GHASH_specific_fields IMB_JOB::GHASH
+
+

GHASH specific fields

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_AES_GMAC_specific_fields IMB_JOB::GMAC
+
+

AES-GMAC specific fields

+ +
+
+ +

◆ hash_alg

+ +
+
+ + + + +
IMB_HASH_ALG IMB_JOB::hash_alg
+
+

Hashing algorithm

+ +
+
+ +

◆ hash_func

+ +
+
+ + + + +
int(* IMB_JOB::hash_func) (struct IMB_JOB *)
+
+

Customer hash function

+ +
+
+ +

◆ hash_start_src_offset_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::hash_start_src_offset_in_bytes
+
+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_HMAC_specific_fields IMB_JOB::HMAC
+
+

HMAC specific fields

+ +
+
+ +

◆ iv

+ +
+
+ + + + +
const uint8_t* IMB_JOB::iv
+
+

Initialization Vector (IV)

+ +
+
+ +

◆ iv_len_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::iv_len_in_bytes
+
+

IV length in bytes

+

Authentication IV length in bytes

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_KASUMI_UIA1_specific_fields IMB_JOB::KASUMI_UIA1
+
+

KASUMI-UIA2 specific fields

+ +
+
+ +

◆ key_len_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::key_len_in_bytes
+
+

Key length in bytes

+ +
+
+ +

◆ msg_len_to_cipher_in_bits

+ +
+
+ + + + +
uint64_t IMB_JOB::msg_len_to_cipher_in_bits
+
+

Length of message to cipher (in bits)

+ +
+
+ +

◆ msg_len_to_cipher_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::msg_len_to_cipher_in_bytes
+
+

Length of message to cipher (in bytes)

+ +
+
+ +

◆ msg_len_to_hash_in_bits

+ +
+
+ + + + +
uint64_t IMB_JOB::msg_len_to_hash_in_bits
+
+

Length of message to hash (in bits)

+ +
+
+ +

◆ msg_len_to_hash_in_bytes

+ +
+
+ + + + +
uint64_t IMB_JOB::msg_len_to_hash_in_bytes
+
+

Length of message to hash (in bytes)

+ +
+
+ +

◆ next_iv

+ +
+
+ + + + +
void* IMB_JOB::next_iv
+
+

Pointer to next IV (last ciphertext block)

+ +
+
+ +

◆ num_sgl_io_segs

+ +
+
+ + + + +
uint64_t IMB_JOB::num_sgl_io_segs
+
+

Number of input/output SGL segments

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_POLY1305_specific_fields IMB_JOB::POLY1305
+
+

Poly1305 specific fields

+ +
+
+ +

◆ reserved

+ +
+
+ + + + +
void* IMB_JOB::reserved
+
+

Reserved bytes

+ +
+
+ +

◆ session_id

+ +
+
+ + + + +
uint32_t IMB_JOB::session_id
+
+

see imb_set_session()

+ +
+
+ +

◆ sgl_io_segs

+ +
+
+ + + + +
const struct IMB_SGL_IOV* IMB_JOB::sgl_io_segs
+
+

Pointer to array of input/output SGL segments

+ +
+
+ +

◆ sgl_state

+ +
+
+ + + + +
IMB_SGL_STATE IMB_JOB::sgl_state
+
+

SGL state (IMB_SGL_INIT/IMB_SGL_UPDATE/IMB_SGL_COMPLETE/ IMB_SGL_ALL)

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_SNOW3G_UIA2_specific_fields IMB_JOB::SNOW3G_UIA2
+
+

SNOW3G-UIA2 specific fields

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_SNOW_V_AEAD_specific_fields IMB_JOB::SNOW_V_AEAD
+
+

SNOW-V AEAD specific fields

+ +
+
+ +

◆ src

+ +
+
+ + + + +
const uint8_t* IMB_JOB::src
+
+

Input buffer. May be ciphertext or plaintext. In-place ciphering allowed.

+ +
+
+ +

◆ status

+ +
+
+ + + + +
IMB_STATUS IMB_JOB::status
+
+

Job status

+ +
+
+ +

◆ suite_id

+ +
+
+ + + + +
void* IMB_JOB::suite_id[4]
+
+

see imb_set_session()

+ +
+
+ +

◆ [union]

+ +
+
+ + + + +
union { ... } IMB_JOB::u
+
+

Hash algorithm-specific fields

+ +
+
+ +

◆ user_data

+ +
+
+ + + + +
void* IMB_JOB::user_data
+
+

Pointer 1 to user data

+ +
+
+ +

◆ user_data2

+ +
+
+ + + + +
void* IMB_JOB::user_data2
+
+

Pointer 2 to user data

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_AES_XCBC_specific_fields IMB_JOB::XCBC
+
+

AES-XCBC specific fields

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } ::_ZUC_EIA3_specific_fields IMB_JOB::ZUC_EIA3
+
+

ZUC-EIA3 specific fields

+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structIMB__JOB.js b/docs/structIMB__JOB.js new file mode 100644 index 0000000000000000000000000000000000000000..807eb993f020aeea0f2f11c98d620f532e56fa9d --- /dev/null +++ b/docs/structIMB__JOB.js @@ -0,0 +1,71 @@ +var structIMB__JOB = +[ + [ "_hashed_auth_key_xor_ipad", "structIMB__JOB.html#a2e74bbeeed2c70f97e6b30a0a12aec0e", null ], + [ "_hashed_auth_key_xor_opad", "structIMB__JOB.html#a684b751d0a275a28d345a3d1051204ef", null ], + [ "_init_tag", "structIMB__JOB.html#a1e2fc881ce9c84173168df1dbda851f8", null ], + [ "_iv", "structIMB__JOB.html#ae571147592c332cea0d90a36e133e6a5", null ], + [ "_iv", "structIMB__JOB.html#a4415e6851e770734c5ec1b46a828ce55", null ], + [ "_iv23", "structIMB__JOB.html#a2bd093b1609dad73bbef2a728995d12f", null ], + [ "_k1_expanded", "structIMB__JOB.html#a76d55a988c71e40c577c6f8f5d0496de", null ], + [ "_k2", "structIMB__JOB.html#adc47d4af15c88dcb7abf01bb07e79e63", null ], + [ "_k3", "structIMB__JOB.html#afa62af4d5f1ecbb80f5216ec298fcfe8", null ], + [ "_key", "structIMB__JOB.html#a7ba1fdbcc6ed2e3840990d2e34dac8e8", null ], + [ "_key", "structIMB__JOB.html#a7a62dc4b2ef35231a93ce035561c4153", null ], + [ "_key", "structIMB__JOB.html#a3b3f616759b854553a351e6b44e08471", null ], + [ "_key_expanded", "structIMB__JOB.html#a95cc77e0c079f43dec1a46a2b75422a0", null ], + [ "_skey1", "structIMB__JOB.html#ab64e8f6bf8dfaf56305b9023103ea5e2", null ], + [ "_skey2", "structIMB__JOB.html#a8e8143e461eab3b2b04333c5f1c17a82", null ], + [ "aad", "structIMB__JOB.html#a4c943fcac7f5c4f5d6d030547840071e", null ], + [ "aad_len_in_bytes", "structIMB__JOB.html#aa607a780bc2c8509270625c7520a11f1", null ], + [ "auth_tag_output", "structIMB__JOB.html#ac37080ddb0baf56c9589e65960430b32", null ], + [ "auth_tag_output_len_in_bytes", "structIMB__JOB.html#a750df4fa7330e2387138409d0eb50e0f", null ], + [ "CBCS", "structIMB__JOB.html#a832c73263792bf8895cf028035730c4e", null ], + [ "CCM", "structIMB__JOB.html#ad43a9c9149ec16b8e9164771ec38d379", null ], + [ "CHACHA20_POLY1305", "structIMB__JOB.html#ace537bcd7b871fddd4514b7fd4f678a3", null ], + [ "chain_order", "structIMB__JOB.html#a41e9fa248ebb040ce1e074a2444b55d2", null ], + [ "cipher_direction", "structIMB__JOB.html#a1449c51b88ccdb2f855525b89c05eb84", null ], + [ "cipher_fields", "structIMB__JOB.html#a006bf024dd3e809fad23c3df813c6fb2", null ], + [ "cipher_func", "structIMB__JOB.html#a652eb3eee14e59be703bf28e69ca0671", null ], + [ "cipher_mode", "structIMB__JOB.html#a78113e875fdb681e775fd32ee67c7597", null ], + [ "cipher_start_offset_in_bits", "structIMB__JOB.html#a60a01da0056367a6f0a326265e4da4fb", null ], + [ "cipher_start_src_offset_in_bits", "structIMB__JOB.html#af5c226d1c875c4e30a8f7bd7c03f05b1", null ], + [ "cipher_start_src_offset_in_bytes", "structIMB__JOB.html#adda0b9bd22d445402bb9866e423707c4", null ], + [ "CMAC", "structIMB__JOB.html#a499c1e90c9004ab7bb50fac98b6a9048", null ], + [ "ctx", "structIMB__JOB.html#a053ceeb65e611c48a97a3e10d03d47eb", null ], + [ "ctx", "structIMB__JOB.html#a4abdf3311d861979dc39669b3b7f1ed8", null ], + [ "dec_keys", "structIMB__JOB.html#af78a3df5a9f79f329c8d1095b9ecea05", null ], + [ "dst", "structIMB__JOB.html#ac1f7c363e7709cf1a7de1364aa8625e2", null ], + [ "enc_keys", "structIMB__JOB.html#ac656e2d98f36bd5547cc9fa297c1de62", null ], + [ "GCM", "structIMB__JOB.html#ac9fc7aef831cfc1ca598a55b250dd858", null ], + [ "GHASH", "structIMB__JOB.html#af24c379b5414a5d3ec290954b98b1c72", null ], + [ "GMAC", "structIMB__JOB.html#a07832e4571ec56f377dffa477b7aafe0", null ], + [ "hash_alg", "structIMB__JOB.html#aa4d9b62d3353150aba88aec5d8d08853", null ], + [ "hash_func", "structIMB__JOB.html#a62883bf8707d09816753832bef5d2902", null ], + [ "hash_start_src_offset_in_bytes", "structIMB__JOB.html#af7cdc3d086f6e2b12d21c1383b2bd152", null ], + [ "HMAC", "structIMB__JOB.html#aa84f5eefd378ab65ebe0b302ccb0df92", null ], + [ "iv", "structIMB__JOB.html#a960fdd62c1c10d2d3cc0a765c0a6531d", null ], + [ "iv_len_in_bytes", "structIMB__JOB.html#a25b4ce8aec20f64cfc393b245d62a09e", null ], + [ "KASUMI_UIA1", "structIMB__JOB.html#a11f452fba9d5cad2b2cdb929b7f244e9", null ], + [ "key_len_in_bytes", "structIMB__JOB.html#abdd8a58a37e8d2daa1c2d333b826a65c", null ], + [ "msg_len_to_cipher_in_bits", "structIMB__JOB.html#a425d57ff565bb2ff76e2b0351b00c751", null ], + [ "msg_len_to_cipher_in_bytes", "structIMB__JOB.html#a9c2e008fdb76df46a33300f692f92265", null ], + [ "msg_len_to_hash_in_bits", "structIMB__JOB.html#ae45d78dcef87042ffa1daf951f715d42", null ], + [ "msg_len_to_hash_in_bytes", "structIMB__JOB.html#a40caf02dcd7e45dd91cfae33159ca26e", null ], + [ "next_iv", "structIMB__JOB.html#a5d473d9cb01f8e2818a6f5f0c9483bd7", null ], + [ "num_sgl_io_segs", "structIMB__JOB.html#a1d0a6baa8d8f7d9714fe86fe7085b8bb", null ], + [ "POLY1305", "structIMB__JOB.html#a15f9c1a5ab25dca6e3a550ff00ad2419", null ], + [ "reserved", "structIMB__JOB.html#a9d5e67861edf38dfa23160b650c7caa9", null ], + [ "session_id", "structIMB__JOB.html#ab4571657aa5147f768083e8969ea96c9", null ], + [ "sgl_io_segs", "structIMB__JOB.html#a2462f9efb16c31ae615df7493dddb343", null ], + [ "sgl_state", "structIMB__JOB.html#a3d7e363be004999d02db464c721e9751", null ], + [ "SNOW3G_UIA2", "structIMB__JOB.html#a2a48f64b9344d0b6ca6775a4fdbe21b7", null ], + [ "SNOW_V_AEAD", "structIMB__JOB.html#abf4a3f2db05c6583de5a99d9590ddca1", null ], + [ "src", "structIMB__JOB.html#af83d63eb71a4f26b4915562bea554f28", null ], + [ "status", "structIMB__JOB.html#ad4e08611eee5188cb84cb2dc337ef9f0", null ], + [ "suite_id", "structIMB__JOB.html#a2e8f1fefae0b0fe245eb20e5cc2aa830", null ], + [ "u", "structIMB__JOB.html#a5e6d8702069bf88ce81cd750df66bcbc", null ], + [ "user_data", "structIMB__JOB.html#abd56907c9485830817f7b30870726aeb", null ], + [ "user_data2", "structIMB__JOB.html#a4e68c8cf14552db625e26477650dfb8b", null ], + [ "XCBC", "structIMB__JOB.html#a3508ac7d49477dffb93b6b9cfbf0ebf3", null ], + [ "ZUC_EIA3", "structIMB__JOB.html#ab42f79355b9173c63aca9b0b577a2b61", null ] +]; \ No newline at end of file diff --git a/docs/structIMB__MGR.html b/docs/structIMB__MGR.html new file mode 100644 index 0000000000000000000000000000000000000000..5da906fedf735f26ff8a03499654510f1e9f2f89 --- /dev/null +++ b/docs/structIMB__MGR.html @@ -0,0 +1,2820 @@ + + + + + + + +intel-ipsec-mb: IMB_MGR Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
IMB_MGR Struct Reference
+
+
+ +

#include <intel-ipsec-mb.h>

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Data Fields

uint64_t flags
 
uint64_t features
 
uint64_t reserved [5]
 
uint32_t used_arch
 
int imb_errno
 
get_next_job_t get_next_job
 
submit_job_t submit_job
 
submit_job_t submit_job_nocheck
 
get_completed_job_t get_completed_job
 
flush_job_t flush_job
 
queue_size_t queue_size
 
keyexp_t keyexp_128
 
keyexp_t keyexp_192
 
keyexp_t keyexp_256
 
cmac_subkey_gen_t cmac_subkey_gen_128
 
xcbc_keyexp_t xcbc_keyexp
 
des_keysched_t des_key_sched
 
hash_one_block_t sha1_one_block
 
hash_one_block_t sha224_one_block
 
hash_one_block_t sha256_one_block
 
hash_one_block_t sha384_one_block
 
hash_one_block_t sha512_one_block
 
hash_one_block_t md5_one_block
 
hash_fn_t sha1
 
hash_fn_t sha224
 
hash_fn_t sha256
 
hash_fn_t sha384
 
hash_fn_t sha512
 
aes_cfb_t aes128_cfb_one
 
aes_gcm_enc_dec_t gcm128_enc
 
aes_gcm_enc_dec_t gcm192_enc
 
aes_gcm_enc_dec_t gcm256_enc
 
aes_gcm_enc_dec_t gcm128_dec
 
aes_gcm_enc_dec_t gcm192_dec
 
aes_gcm_enc_dec_t gcm256_dec
 
aes_gcm_init_t gcm128_init
 
aes_gcm_init_t gcm192_init
 
aes_gcm_init_t gcm256_init
 
aes_gcm_enc_dec_update_t gcm128_enc_update
 
aes_gcm_enc_dec_update_t gcm192_enc_update
 
aes_gcm_enc_dec_update_t gcm256_enc_update
 
aes_gcm_enc_dec_update_t gcm128_dec_update
 
aes_gcm_enc_dec_update_t gcm192_dec_update
 
aes_gcm_enc_dec_update_t gcm256_dec_update
 
aes_gcm_enc_dec_finalize_t gcm128_enc_finalize
 
aes_gcm_enc_dec_finalize_t gcm192_enc_finalize
 
aes_gcm_enc_dec_finalize_t gcm256_enc_finalize
 
aes_gcm_enc_dec_finalize_t gcm128_dec_finalize
 
aes_gcm_enc_dec_finalize_t gcm192_dec_finalize
 
aes_gcm_enc_dec_finalize_t gcm256_dec_finalize
 
aes_gcm_precomp_t gcm128_precomp
 
aes_gcm_precomp_t gcm192_precomp
 
aes_gcm_precomp_t gcm256_precomp
 
aes_gcm_pre_t gcm128_pre
 
aes_gcm_pre_t gcm192_pre
 
aes_gcm_pre_t gcm256_pre
 
zuc_eea3_1_buffer_t eea3_1_buffer
 
zuc_eea3_4_buffer_t eea3_4_buffer
 
zuc_eea3_n_buffer_t eea3_n_buffer
 
zuc_eia3_1_buffer_t eia3_1_buffer
 
kasumi_f8_1_buffer_t f8_1_buffer
 
kasumi_f8_1_buffer_bit_t f8_1_buffer_bit
 
kasumi_f8_2_buffer_t f8_2_buffer
 
kasumi_f8_3_buffer_t f8_3_buffer
 
kasumi_f8_4_buffer_t f8_4_buffer
 
kasumi_f8_n_buffer_t f8_n_buffer
 
kasumi_f9_1_buffer_t f9_1_buffer
 
kasumi_f9_1_buffer_user_t f9_1_buffer_user
 
kasumi_init_f8_key_sched_t kasumi_init_f8_key_sched
 
kasumi_init_f9_key_sched_t kasumi_init_f9_key_sched
 
kasumi_key_sched_size_t kasumi_key_sched_size
 
snow3g_f8_1_buffer_bit_t snow3g_f8_1_buffer_bit
 
snow3g_f8_1_buffer_t snow3g_f8_1_buffer
 
snow3g_f8_2_buffer_t snow3g_f8_2_buffer
 
snow3g_f8_4_buffer_t snow3g_f8_4_buffer
 
snow3g_f8_8_buffer_t snow3g_f8_8_buffer
 
snow3g_f8_n_buffer_t snow3g_f8_n_buffer
 
snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey
 
snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey
 
snow3g_f9_1_buffer_t snow3g_f9_1_buffer
 
snow3g_init_key_sched_t snow3g_init_key_sched
 
snow3g_key_sched_size_t snow3g_key_sched_size
 
ghash_t ghash
 
zuc_eia3_n_buffer_t eia3_n_buffer
 
aes_gcm_init_var_iv_t gcm128_init_var_iv
 
aes_gcm_init_var_iv_t gcm192_init_var_iv
 
aes_gcm_init_var_iv_t gcm256_init_var_iv
 
aes_gmac_init_t gmac128_init
 
aes_gmac_init_t gmac192_init
 
aes_gmac_init_t gmac256_init
 
aes_gmac_update_t gmac128_update
 
aes_gmac_update_t gmac192_update
 
aes_gmac_update_t gmac256_update
 
aes_gmac_finalize_t gmac128_finalize
 
aes_gmac_finalize_t gmac192_finalize
 
aes_gmac_finalize_t gmac256_finalize
 
hec_32_t hec_32
 
hec_64_t hec_64
 
cmac_subkey_gen_t cmac_subkey_gen_256
 
aes_gcm_pre_t ghash_pre
 
crc32_fn_t crc32_ethernet_fcs
 
crc32_fn_t crc16_x25
 
crc32_fn_t crc32_sctp
 
crc32_fn_t crc24_lte_a
 
crc32_fn_t crc24_lte_b
 
crc32_fn_t crc16_fp_data
 
crc32_fn_t crc11_fp_header
 
crc32_fn_t crc7_fp_header
 
crc32_fn_t crc10_iuup_data
 
crc32_fn_t crc6_iuup_header
 
crc32_fn_t crc32_wimax_ofdma_data
 
crc32_fn_t crc8_wimax_ofdma_hcs
 
chacha_poly_init_t chacha20_poly1305_init
 
chacha_poly_enc_dec_update_t chacha20_poly1305_enc_update
 
chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update
 
chacha_poly_finalize_t chacha20_poly1305_finalize
 
burst_fn_t get_next_burst
 
burst_fn_t submit_burst
 
burst_fn_t submit_burst_nocheck
 
burst_fn_t flush_burst
 
submit_cipher_burst_t submit_cipher_burst
 
submit_cipher_burst_t submit_cipher_burst_nocheck
 
submit_hash_burst_t submit_hash_burst
 
submit_hash_burst_t submit_hash_burst_nocheck
 
aes_cfb_t aes256_cfb_one
 
aes_ecb_quic_t aes_ecb_128_quic
 
aes_ecb_quic_t aes_ecb_256_quic
 
void(* set_suite_id )(struct IMB_MGR *, IMB_JOB *)
 
int earliest_job
 
int next_job
 
IMB_JOB jobs [IMB_MAX_JOBS]
 
void * aes128_ooo
 
void * aes192_ooo
 
void * aes256_ooo
 
void * docsis128_sec_ooo
 
void * docsis128_crc32_sec_ooo
 
void * docsis256_sec_ooo
 
void * docsis256_crc32_sec_ooo
 
void * des_enc_ooo
 
void * des_dec_ooo
 
void * des3_enc_ooo
 
void * des3_dec_ooo
 
void * docsis_des_enc_ooo
 
void * docsis_des_dec_ooo
 
void * hmac_sha_1_ooo
 
void * hmac_sha_224_ooo
 
void * hmac_sha_256_ooo
 
void * hmac_sha_384_ooo
 
void * hmac_sha_512_ooo
 
void * hmac_md5_ooo
 
void * aes_xcbc_ooo
 
void * aes_ccm_ooo
 
void * aes_cmac_ooo
 
void * zuc_eea3_ooo
 
void * zuc_eia3_ooo
 
void * aes128_cbcs_ooo
 
void * zuc256_eea3_ooo
 
void * zuc256_eia3_ooo
 
void * aes256_ccm_ooo
 
void * aes256_cmac_ooo
 
void * snow3g_uea2_ooo
 
void * snow3g_uia2_ooo
 
void * sha_1_ooo
 
void * sha_224_ooo
 
void * sha_256_ooo
 
void * sha_384_ooo
 
void * sha_512_ooo
 
void * end_ooo
 
+

Field Documentation

+ +

◆ aes128_cbcs_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes128_cbcs_ooo
+
+ +
+
+ +

◆ aes128_cfb_one

+ +
+
+ + + + +
aes_cfb_t IMB_MGR::aes128_cfb_one
+
+ +
+
+ +

◆ aes128_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes128_ooo
+
+ +
+
+ +

◆ aes192_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes192_ooo
+
+ +
+
+ +

◆ aes256_ccm_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes256_ccm_ooo
+
+ +
+
+ +

◆ aes256_cfb_one

+ +
+
+ + + + +
aes_cfb_t IMB_MGR::aes256_cfb_one
+
+ +
+
+ +

◆ aes256_cmac_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes256_cmac_ooo
+
+ +
+
+ +

◆ aes256_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes256_ooo
+
+ +
+
+ +

◆ aes_ccm_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes_ccm_ooo
+
+ +
+
+ +

◆ aes_cmac_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes_cmac_ooo
+
+ +
+
+ +

◆ aes_ecb_128_quic

+ +
+
+ + + + +
aes_ecb_quic_t IMB_MGR::aes_ecb_128_quic
+
+ +
+
+ +

◆ aes_ecb_256_quic

+ +
+
+ + + + +
aes_ecb_quic_t IMB_MGR::aes_ecb_256_quic
+
+ +
+
+ +

◆ aes_xcbc_ooo

+ +
+
+ + + + +
void* IMB_MGR::aes_xcbc_ooo
+
+ +
+
+ +

◆ chacha20_poly1305_dec_update

+ +
+
+ + + + +
chacha_poly_enc_dec_update_t IMB_MGR::chacha20_poly1305_dec_update
+
+ +
+
+ +

◆ chacha20_poly1305_enc_update

+ +
+
+ + + + +
chacha_poly_enc_dec_update_t IMB_MGR::chacha20_poly1305_enc_update
+
+ +
+
+ +

◆ chacha20_poly1305_finalize

+ +
+
+ + + + +
chacha_poly_finalize_t IMB_MGR::chacha20_poly1305_finalize
+
+ +
+
+ +

◆ chacha20_poly1305_init

+ +
+
+ + + + +
chacha_poly_init_t IMB_MGR::chacha20_poly1305_init
+
+ +
+
+ +

◆ cmac_subkey_gen_128

+ +
+
+ + + + +
cmac_subkey_gen_t IMB_MGR::cmac_subkey_gen_128
+
+ +
+
+ +

◆ cmac_subkey_gen_256

+ +
+
+ + + + +
cmac_subkey_gen_t IMB_MGR::cmac_subkey_gen_256
+
+ +
+
+ +

◆ crc10_iuup_data

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc10_iuup_data
+
+ +
+
+ +

◆ crc11_fp_header

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc11_fp_header
+
+ +
+
+ +

◆ crc16_fp_data

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc16_fp_data
+
+ +
+
+ +

◆ crc16_x25

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc16_x25
+
+ +
+
+ +

◆ crc24_lte_a

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc24_lte_a
+
+ +
+
+ +

◆ crc24_lte_b

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc24_lte_b
+
+ +
+
+ +

◆ crc32_ethernet_fcs

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc32_ethernet_fcs
+
+ +
+
+ +

◆ crc32_sctp

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc32_sctp
+
+ +
+
+ +

◆ crc32_wimax_ofdma_data

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc32_wimax_ofdma_data
+
+ +
+
+ +

◆ crc6_iuup_header

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc6_iuup_header
+
+ +
+
+ +

◆ crc7_fp_header

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc7_fp_header
+
+ +
+
+ +

◆ crc8_wimax_ofdma_hcs

+ +
+
+ + + + +
crc32_fn_t IMB_MGR::crc8_wimax_ofdma_hcs
+
+ +
+
+ +

◆ des3_dec_ooo

+ +
+
+ + + + +
void* IMB_MGR::des3_dec_ooo
+
+ +
+
+ +

◆ des3_enc_ooo

+ +
+
+ + + + +
void* IMB_MGR::des3_enc_ooo
+
+ +
+
+ +

◆ des_dec_ooo

+ +
+
+ + + + +
void* IMB_MGR::des_dec_ooo
+
+ +
+
+ +

◆ des_enc_ooo

+ +
+
+ + + + +
void* IMB_MGR::des_enc_ooo
+
+ +
+
+ +

◆ des_key_sched

+ +
+
+ + + + +
des_keysched_t IMB_MGR::des_key_sched
+
+ +
+
+ +

◆ docsis128_crc32_sec_ooo

+ +
+
+ + + + +
void* IMB_MGR::docsis128_crc32_sec_ooo
+
+ +
+
+ +

◆ docsis128_sec_ooo

+ +
+
+ + + + +
void* IMB_MGR::docsis128_sec_ooo
+
+ +
+
+ +

◆ docsis256_crc32_sec_ooo

+ +
+
+ + + + +
void* IMB_MGR::docsis256_crc32_sec_ooo
+
+ +
+
+ +

◆ docsis256_sec_ooo

+ +
+
+ + + + +
void* IMB_MGR::docsis256_sec_ooo
+
+ +
+
+ +

◆ docsis_des_dec_ooo

+ +
+
+ + + + +
void* IMB_MGR::docsis_des_dec_ooo
+
+ +
+
+ +

◆ docsis_des_enc_ooo

+ +
+
+ + + + +
void* IMB_MGR::docsis_des_enc_ooo
+
+ +
+
+ +

◆ earliest_job

+ +
+
+ + + + +
int IMB_MGR::earliest_job
+
+

byte offset, -1 if none

+ +
+
+ +

◆ eea3_1_buffer

+ +
+
+ + + + +
zuc_eea3_1_buffer_t IMB_MGR::eea3_1_buffer
+
+ +
+
+ +

◆ eea3_4_buffer

+ +
+
+ + + + +
zuc_eea3_4_buffer_t IMB_MGR::eea3_4_buffer
+
+ +
+
+ +

◆ eea3_n_buffer

+ +
+
+ + + + +
zuc_eea3_n_buffer_t IMB_MGR::eea3_n_buffer
+
+ +
+
+ +

◆ eia3_1_buffer

+ +
+
+ + + + +
zuc_eia3_1_buffer_t IMB_MGR::eia3_1_buffer
+
+ +
+
+ +

◆ eia3_n_buffer

+ +
+
+ + + + +
zuc_eia3_n_buffer_t IMB_MGR::eia3_n_buffer
+
+ +
+
+ +

◆ end_ooo

+ +
+
+ + + + +
void* IMB_MGR::end_ooo
+
+ +
+
+ +

◆ f8_1_buffer

+ +
+
+ + + + +
kasumi_f8_1_buffer_t IMB_MGR::f8_1_buffer
+
+ +
+
+ +

◆ f8_1_buffer_bit

+ +
+
+ + + + +
kasumi_f8_1_buffer_bit_t IMB_MGR::f8_1_buffer_bit
+
+ +
+
+ +

◆ f8_2_buffer

+ +
+
+ + + + +
kasumi_f8_2_buffer_t IMB_MGR::f8_2_buffer
+
+ +
+
+ +

◆ f8_3_buffer

+ +
+
+ + + + +
kasumi_f8_3_buffer_t IMB_MGR::f8_3_buffer
+
+ +
+
+ +

◆ f8_4_buffer

+ +
+
+ + + + +
kasumi_f8_4_buffer_t IMB_MGR::f8_4_buffer
+
+ +
+
+ +

◆ f8_n_buffer

+ +
+
+ + + + +
kasumi_f8_n_buffer_t IMB_MGR::f8_n_buffer
+
+ +
+
+ +

◆ f9_1_buffer

+ +
+
+ + + + +
kasumi_f9_1_buffer_t IMB_MGR::f9_1_buffer
+
+ +
+
+ +

◆ f9_1_buffer_user

+ +
+
+ + + + +
kasumi_f9_1_buffer_user_t IMB_MGR::f9_1_buffer_user
+
+ +
+
+ +

◆ features

+ +
+
+ + + + +
uint64_t IMB_MGR::features
+
+

reflects features of multi-buffer instance

+ +
+
+ +

◆ flags

+ +
+
+ + + + +
uint64_t IMB_MGR::flags
+
+

passed to alloc_mb_mgr()

+ +
+
+ +

◆ flush_burst

+ +
+
+ + + + +
burst_fn_t IMB_MGR::flush_burst
+
+ +
+
+ +

◆ flush_job

+ +
+
+ + + + +
flush_job_t IMB_MGR::flush_job
+
+ +
+
+ +

◆ gcm128_dec

+ +
+
+ + + + +
aes_gcm_enc_dec_t IMB_MGR::gcm128_dec
+
+ +
+
+ +

◆ gcm128_dec_finalize

+ +
+
+ + + + +
aes_gcm_enc_dec_finalize_t IMB_MGR::gcm128_dec_finalize
+
+ +
+
+ +

◆ gcm128_dec_update

+ +
+
+ + + + +
aes_gcm_enc_dec_update_t IMB_MGR::gcm128_dec_update
+
+ +
+
+ +

◆ gcm128_enc

+ +
+
+ + + + +
aes_gcm_enc_dec_t IMB_MGR::gcm128_enc
+
+ +
+
+ +

◆ gcm128_enc_finalize

+ +
+
+ + + + +
aes_gcm_enc_dec_finalize_t IMB_MGR::gcm128_enc_finalize
+
+ +
+
+ +

◆ gcm128_enc_update

+ +
+
+ + + + +
aes_gcm_enc_dec_update_t IMB_MGR::gcm128_enc_update
+
+ +
+
+ +

◆ gcm128_init

+ +
+
+ + + + +
aes_gcm_init_t IMB_MGR::gcm128_init
+
+ +
+
+ +

◆ gcm128_init_var_iv

+ +
+
+ + + + +
aes_gcm_init_var_iv_t IMB_MGR::gcm128_init_var_iv
+
+ +
+
+ +

◆ gcm128_pre

+ +
+
+ + + + +
aes_gcm_pre_t IMB_MGR::gcm128_pre
+
+ +
+
+ +

◆ gcm128_precomp

+ +
+
+ + + + +
aes_gcm_precomp_t IMB_MGR::gcm128_precomp
+
+ +
+
+ +

◆ gcm192_dec

+ +
+
+ + + + +
aes_gcm_enc_dec_t IMB_MGR::gcm192_dec
+
+ +
+
+ +

◆ gcm192_dec_finalize

+ +
+
+ + + + +
aes_gcm_enc_dec_finalize_t IMB_MGR::gcm192_dec_finalize
+
+ +
+
+ +

◆ gcm192_dec_update

+ +
+
+ + + + +
aes_gcm_enc_dec_update_t IMB_MGR::gcm192_dec_update
+
+ +
+
+ +

◆ gcm192_enc

+ +
+
+ + + + +
aes_gcm_enc_dec_t IMB_MGR::gcm192_enc
+
+ +
+
+ +

◆ gcm192_enc_finalize

+ +
+
+ + + + +
aes_gcm_enc_dec_finalize_t IMB_MGR::gcm192_enc_finalize
+
+ +
+
+ +

◆ gcm192_enc_update

+ +
+
+ + + + +
aes_gcm_enc_dec_update_t IMB_MGR::gcm192_enc_update
+
+ +
+
+ +

◆ gcm192_init

+ +
+
+ + + + +
aes_gcm_init_t IMB_MGR::gcm192_init
+
+ +
+
+ +

◆ gcm192_init_var_iv

+ +
+
+ + + + +
aes_gcm_init_var_iv_t IMB_MGR::gcm192_init_var_iv
+
+ +
+
+ +

◆ gcm192_pre

+ +
+
+ + + + +
aes_gcm_pre_t IMB_MGR::gcm192_pre
+
+ +
+
+ +

◆ gcm192_precomp

+ +
+
+ + + + +
aes_gcm_precomp_t IMB_MGR::gcm192_precomp
+
+ +
+
+ +

◆ gcm256_dec

+ +
+
+ + + + +
aes_gcm_enc_dec_t IMB_MGR::gcm256_dec
+
+ +
+
+ +

◆ gcm256_dec_finalize

+ +
+
+ + + + +
aes_gcm_enc_dec_finalize_t IMB_MGR::gcm256_dec_finalize
+
+ +
+
+ +

◆ gcm256_dec_update

+ +
+
+ + + + +
aes_gcm_enc_dec_update_t IMB_MGR::gcm256_dec_update
+
+ +
+
+ +

◆ gcm256_enc

+ +
+
+ + + + +
aes_gcm_enc_dec_t IMB_MGR::gcm256_enc
+
+ +
+
+ +

◆ gcm256_enc_finalize

+ +
+
+ + + + +
aes_gcm_enc_dec_finalize_t IMB_MGR::gcm256_enc_finalize
+
+ +
+
+ +

◆ gcm256_enc_update

+ +
+
+ + + + +
aes_gcm_enc_dec_update_t IMB_MGR::gcm256_enc_update
+
+ +
+
+ +

◆ gcm256_init

+ +
+
+ + + + +
aes_gcm_init_t IMB_MGR::gcm256_init
+
+ +
+
+ +

◆ gcm256_init_var_iv

+ +
+
+ + + + +
aes_gcm_init_var_iv_t IMB_MGR::gcm256_init_var_iv
+
+ +
+
+ +

◆ gcm256_pre

+ +
+
+ + + + +
aes_gcm_pre_t IMB_MGR::gcm256_pre
+
+ +
+
+ +

◆ gcm256_precomp

+ +
+
+ + + + +
aes_gcm_precomp_t IMB_MGR::gcm256_precomp
+
+ +
+
+ +

◆ get_completed_job

+ +
+
+ + + + +
get_completed_job_t IMB_MGR::get_completed_job
+
+ +
+
+ +

◆ get_next_burst

+ +
+
+ + + + +
burst_fn_t IMB_MGR::get_next_burst
+
+ +
+
+ +

◆ get_next_job

+ +
+
+ + + + +
get_next_job_t IMB_MGR::get_next_job
+
+

ARCH handlers / API Careful as changes here can break ABI compatibility (always include function pointers at the end of the list, before "earliest_job")

+ +
+
+ +

◆ ghash

+ +
+
+ + + + +
ghash_t IMB_MGR::ghash
+
+ +
+
+ +

◆ ghash_pre

+ +
+
+ + + + +
aes_gcm_pre_t IMB_MGR::ghash_pre
+
+ +
+
+ +

◆ gmac128_finalize

+ +
+
+ + + + +
aes_gmac_finalize_t IMB_MGR::gmac128_finalize
+
+ +
+
+ +

◆ gmac128_init

+ +
+
+ + + + +
aes_gmac_init_t IMB_MGR::gmac128_init
+
+ +
+
+ +

◆ gmac128_update

+ +
+
+ + + + +
aes_gmac_update_t IMB_MGR::gmac128_update
+
+ +
+
+ +

◆ gmac192_finalize

+ +
+
+ + + + +
aes_gmac_finalize_t IMB_MGR::gmac192_finalize
+
+ +
+
+ +

◆ gmac192_init

+ +
+
+ + + + +
aes_gmac_init_t IMB_MGR::gmac192_init
+
+ +
+
+ +

◆ gmac192_update

+ +
+
+ + + + +
aes_gmac_update_t IMB_MGR::gmac192_update
+
+ +
+
+ +

◆ gmac256_finalize

+ +
+
+ + + + +
aes_gmac_finalize_t IMB_MGR::gmac256_finalize
+
+ +
+
+ +

◆ gmac256_init

+ +
+
+ + + + +
aes_gmac_init_t IMB_MGR::gmac256_init
+
+ +
+
+ +

◆ gmac256_update

+ +
+
+ + + + +
aes_gmac_update_t IMB_MGR::gmac256_update
+
+ +
+
+ +

◆ hec_32

+ +
+
+ + + + +
hec_32_t IMB_MGR::hec_32
+
+ +
+
+ +

◆ hec_64

+ +
+
+ + + + +
hec_64_t IMB_MGR::hec_64
+
+ +
+
+ +

◆ hmac_md5_ooo

+ +
+
+ + + + +
void* IMB_MGR::hmac_md5_ooo
+
+ +
+
+ +

◆ hmac_sha_1_ooo

+ +
+
+ + + + +
void* IMB_MGR::hmac_sha_1_ooo
+
+ +
+
+ +

◆ hmac_sha_224_ooo

+ +
+
+ + + + +
void* IMB_MGR::hmac_sha_224_ooo
+
+ +
+
+ +

◆ hmac_sha_256_ooo

+ +
+
+ + + + +
void* IMB_MGR::hmac_sha_256_ooo
+
+ +
+
+ +

◆ hmac_sha_384_ooo

+ +
+
+ + + + +
void* IMB_MGR::hmac_sha_384_ooo
+
+ +
+
+ +

◆ hmac_sha_512_ooo

+ +
+
+ + + + +
void* IMB_MGR::hmac_sha_512_ooo
+
+ +
+
+ +

◆ imb_errno

+ +
+
+ + + + +
int IMB_MGR::imb_errno
+
+

per mb_mgr error status

+ +
+
+ +

◆ jobs

+ +
+
+ + + + +
IMB_JOB IMB_MGR::jobs[IMB_MAX_JOBS]
+
+ +
+
+ +

◆ kasumi_init_f8_key_sched

+ +
+
+ + + + +
kasumi_init_f8_key_sched_t IMB_MGR::kasumi_init_f8_key_sched
+
+ +
+
+ +

◆ kasumi_init_f9_key_sched

+ +
+
+ + + + +
kasumi_init_f9_key_sched_t IMB_MGR::kasumi_init_f9_key_sched
+
+ +
+
+ +

◆ kasumi_key_sched_size

+ +
+
+ + + + +
kasumi_key_sched_size_t IMB_MGR::kasumi_key_sched_size
+
+ +
+
+ +

◆ keyexp_128

+ +
+
+ + + + +
keyexp_t IMB_MGR::keyexp_128
+
+ +
+
+ +

◆ keyexp_192

+ +
+
+ + + + +
keyexp_t IMB_MGR::keyexp_192
+
+ +
+
+ +

◆ keyexp_256

+ +
+
+ + + + +
keyexp_t IMB_MGR::keyexp_256
+
+ +
+
+ +

◆ md5_one_block

+ +
+
+ + + + +
hash_one_block_t IMB_MGR::md5_one_block
+
+ +
+
+ +

◆ next_job

+ +
+
+ + + + +
int IMB_MGR::next_job
+
+

byte offset

+ +
+
+ +

◆ queue_size

+ +
+
+ + + + +
queue_size_t IMB_MGR::queue_size
+
+ +
+
+ +

◆ reserved

+ +
+
+ + + + +
uint64_t IMB_MGR::reserved[5]
+
+

reserved for the future

+ +
+
+ +

◆ set_suite_id

+ +
+
+ + + + +
void(* IMB_MGR::set_suite_id) (struct IMB_MGR *, IMB_JOB *)
+
+ +
+
+ +

◆ sha1

+ +
+
+ + + + +
hash_fn_t IMB_MGR::sha1
+
+ +
+
+ +

◆ sha1_one_block

+ +
+
+ + + + +
hash_one_block_t IMB_MGR::sha1_one_block
+
+ +
+
+ +

◆ sha224

+ +
+
+ + + + +
hash_fn_t IMB_MGR::sha224
+
+ +
+
+ +

◆ sha224_one_block

+ +
+
+ + + + +
hash_one_block_t IMB_MGR::sha224_one_block
+
+ +
+
+ +

◆ sha256

+ +
+
+ + + + +
hash_fn_t IMB_MGR::sha256
+
+ +
+
+ +

◆ sha256_one_block

+ +
+
+ + + + +
hash_one_block_t IMB_MGR::sha256_one_block
+
+ +
+
+ +

◆ sha384

+ +
+
+ + + + +
hash_fn_t IMB_MGR::sha384
+
+ +
+
+ +

◆ sha384_one_block

+ +
+
+ + + + +
hash_one_block_t IMB_MGR::sha384_one_block
+
+ +
+
+ +

◆ sha512

+ +
+
+ + + + +
hash_fn_t IMB_MGR::sha512
+
+ +
+
+ +

◆ sha512_one_block

+ +
+
+ + + + +
hash_one_block_t IMB_MGR::sha512_one_block
+
+ +
+
+ +

◆ sha_1_ooo

+ +
+
+ + + + +
void* IMB_MGR::sha_1_ooo
+
+ +
+
+ +

◆ sha_224_ooo

+ +
+
+ + + + +
void* IMB_MGR::sha_224_ooo
+
+ +
+
+ +

◆ sha_256_ooo

+ +
+
+ + + + +
void* IMB_MGR::sha_256_ooo
+
+ +
+
+ +

◆ sha_384_ooo

+ +
+
+ + + + +
void* IMB_MGR::sha_384_ooo
+
+ +
+
+ +

◆ sha_512_ooo

+ +
+
+ + + + +
void* IMB_MGR::sha_512_ooo
+
+ +
+
+ +

◆ snow3g_f8_1_buffer

+ +
+
+ + + + +
snow3g_f8_1_buffer_t IMB_MGR::snow3g_f8_1_buffer
+
+ +
+
+ +

◆ snow3g_f8_1_buffer_bit

+ +
+
+ + + + +
snow3g_f8_1_buffer_bit_t IMB_MGR::snow3g_f8_1_buffer_bit
+
+ +
+
+ +

◆ snow3g_f8_2_buffer

+ +
+
+ + + + +
snow3g_f8_2_buffer_t IMB_MGR::snow3g_f8_2_buffer
+
+ +
+
+ +

◆ snow3g_f8_4_buffer

+ +
+
+ + + + +
snow3g_f8_4_buffer_t IMB_MGR::snow3g_f8_4_buffer
+
+ +
+
+ +

◆ snow3g_f8_8_buffer

+ +
+
+ + + + +
snow3g_f8_8_buffer_t IMB_MGR::snow3g_f8_8_buffer
+
+ +
+
+ +

◆ snow3g_f8_8_buffer_multikey

+ +
+
+ + + + +
snow3g_f8_8_buffer_multikey_t IMB_MGR::snow3g_f8_8_buffer_multikey
+
+ +
+
+ +

◆ snow3g_f8_n_buffer

+ +
+
+ + + + +
snow3g_f8_n_buffer_t IMB_MGR::snow3g_f8_n_buffer
+
+ +
+
+ +

◆ snow3g_f8_n_buffer_multikey

+ +
+
+ + + + +
snow3g_f8_n_buffer_multikey_t IMB_MGR::snow3g_f8_n_buffer_multikey
+
+ +
+
+ +

◆ snow3g_f9_1_buffer

+ +
+
+ + + + +
snow3g_f9_1_buffer_t IMB_MGR::snow3g_f9_1_buffer
+
+ +
+
+ +

◆ snow3g_init_key_sched

+ +
+
+ + + + +
snow3g_init_key_sched_t IMB_MGR::snow3g_init_key_sched
+
+ +
+
+ +

◆ snow3g_key_sched_size

+ +
+
+ + + + +
snow3g_key_sched_size_t IMB_MGR::snow3g_key_sched_size
+
+ +
+
+ +

◆ snow3g_uea2_ooo

+ +
+
+ + + + +
void* IMB_MGR::snow3g_uea2_ooo
+
+ +
+
+ +

◆ snow3g_uia2_ooo

+ +
+
+ + + + +
void* IMB_MGR::snow3g_uia2_ooo
+
+ +
+
+ +

◆ submit_burst

+ +
+
+ + + + +
burst_fn_t IMB_MGR::submit_burst
+
+ +
+
+ +

◆ submit_burst_nocheck

+ +
+
+ + + + +
burst_fn_t IMB_MGR::submit_burst_nocheck
+
+ +
+
+ +

◆ submit_cipher_burst

+ +
+
+ + + + +
submit_cipher_burst_t IMB_MGR::submit_cipher_burst
+
+ +
+
+ +

◆ submit_cipher_burst_nocheck

+ +
+
+ + + + +
submit_cipher_burst_t IMB_MGR::submit_cipher_burst_nocheck
+
+ +
+
+ +

◆ submit_hash_burst

+ +
+
+ + + + +
submit_hash_burst_t IMB_MGR::submit_hash_burst
+
+ +
+
+ +

◆ submit_hash_burst_nocheck

+ +
+
+ + + + +
submit_hash_burst_t IMB_MGR::submit_hash_burst_nocheck
+
+ +
+
+ +

◆ submit_job

+ +
+
+ + + + +
submit_job_t IMB_MGR::submit_job
+
+ +
+
+ +

◆ submit_job_nocheck

+ +
+
+ + + + +
submit_job_t IMB_MGR::submit_job_nocheck
+
+ +
+
+ +

◆ used_arch

+ +
+
+ + + + +
uint32_t IMB_MGR::used_arch
+
+

Architecture being used

+ +
+
+ +

◆ xcbc_keyexp

+ +
+
+ + + + +
xcbc_keyexp_t IMB_MGR::xcbc_keyexp
+
+ +
+
+ +

◆ zuc256_eea3_ooo

+ +
+
+ + + + +
void* IMB_MGR::zuc256_eea3_ooo
+
+ +
+
+ +

◆ zuc256_eia3_ooo

+ +
+
+ + + + +
void* IMB_MGR::zuc256_eia3_ooo
+
+ +
+
+ +

◆ zuc_eea3_ooo

+ +
+
+ + + + +
void* IMB_MGR::zuc_eea3_ooo
+
+ +
+
+ +

◆ zuc_eia3_ooo

+ +
+
+ + + + +
void* IMB_MGR::zuc_eia3_ooo
+
+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structIMB__MGR.js b/docs/structIMB__MGR.js new file mode 100644 index 0000000000000000000000000000000000000000..e354c1670489bf16ca387d962890db00f44b2f76 --- /dev/null +++ b/docs/structIMB__MGR.js @@ -0,0 +1,171 @@ +var structIMB__MGR = +[ + [ "aes128_cbcs_ooo", "structIMB__MGR.html#a9c72f3341a9a4837595338d06c3eda5c", null ], + [ "aes128_cfb_one", "structIMB__MGR.html#abd468de03563a40da0b21b0885f44cbe", null ], + [ "aes128_ooo", "structIMB__MGR.html#a97fd05a4057fc898f647955a7ba8501b", null ], + [ "aes192_ooo", "structIMB__MGR.html#afe48187946dff59f6c69103501691921", null ], + [ "aes256_ccm_ooo", "structIMB__MGR.html#a04c28bfe763544b68dac98e23aa0a8d9", null ], + [ "aes256_cfb_one", "structIMB__MGR.html#ab4e116827875a9dd2573f425081c7eaf", null ], + [ "aes256_cmac_ooo", "structIMB__MGR.html#acbd18bab8176262daab2bd808ca30616", null ], + [ "aes256_ooo", "structIMB__MGR.html#a6abbf129001cb44003a10105115e52b7", null ], + [ "aes_ccm_ooo", "structIMB__MGR.html#a3eeeead99056e46d8cb1ed316f587aaf", null ], + [ "aes_cmac_ooo", "structIMB__MGR.html#a0bd2acac3a6c3f90a998ed53a7d28e57", null ], + [ "aes_ecb_128_quic", "structIMB__MGR.html#a436adbd14682855df093afb36c569039", null ], + [ "aes_ecb_256_quic", "structIMB__MGR.html#a5e589dea4f677ad01cfb01d06884a281", null ], + [ "aes_xcbc_ooo", "structIMB__MGR.html#ad554a25793a904ae6cf7ca87250adbd3", null ], + [ "chacha20_poly1305_dec_update", "structIMB__MGR.html#a715aede4176564ac475e42c8a377bd8d", null ], + [ "chacha20_poly1305_enc_update", "structIMB__MGR.html#ab2c87c40f9844795e3987536c602ce03", null ], + [ "chacha20_poly1305_finalize", "structIMB__MGR.html#ab18b3fcba2aa252bb31f66093bc62abb", null ], + [ "chacha20_poly1305_init", "structIMB__MGR.html#a67b96b1b26fa25391f9f4b83a7a4b6c0", null ], + [ "cmac_subkey_gen_128", "structIMB__MGR.html#aa3404b7811d57821ea574994e0c88995", null ], + [ "cmac_subkey_gen_256", "structIMB__MGR.html#abd500d2cc03f6470ac6988785bebdf76", null ], + [ "crc10_iuup_data", "structIMB__MGR.html#a7d0979deda4e2ff7ac202c82611ce94c", null ], + [ "crc11_fp_header", "structIMB__MGR.html#a4b901916c2f262f1c873e4cec7b7f474", null ], + [ "crc16_fp_data", "structIMB__MGR.html#ad566ac43002c2ba8ecb93048d5edc41b", null ], + [ "crc16_x25", "structIMB__MGR.html#a52a7ab10d6434820e9b31d0e772ea0bc", null ], + [ "crc24_lte_a", "structIMB__MGR.html#a058cfc60f2fdd68678b36f4f038270ac", null ], + [ "crc24_lte_b", "structIMB__MGR.html#abe8406f3582d3540edb770e153b74c3e", null ], + [ "crc32_ethernet_fcs", "structIMB__MGR.html#a8483109fbe42f8b7fa6e7f53e016d5ec", null ], + [ "crc32_sctp", "structIMB__MGR.html#a4aed66c9e73808a65a062e1a5b0f66aa", null ], + [ "crc32_wimax_ofdma_data", "structIMB__MGR.html#ad08e44bc4edc54c556ea2245f848b974", null ], + [ "crc6_iuup_header", "structIMB__MGR.html#aa610dc7226673020be410762b995b725", null ], + [ "crc7_fp_header", "structIMB__MGR.html#ac819ddcc968604186983d58f275bd24b", null ], + [ "crc8_wimax_ofdma_hcs", "structIMB__MGR.html#a5e5be572c46ed01ac11fc95738373e2a", null ], + [ "des3_dec_ooo", "structIMB__MGR.html#a2ab69fdbef6a9a0574fcbb5ea9a7da3b", null ], + [ "des3_enc_ooo", "structIMB__MGR.html#a87f7566634911b4dba48a19a0c3a4993", null ], + [ "des_dec_ooo", "structIMB__MGR.html#af03e401b002e2e415c8a927c46e1517b", null ], + [ "des_enc_ooo", "structIMB__MGR.html#a54ec50411917f1aa9016db268ac638c1", null ], + [ "des_key_sched", "structIMB__MGR.html#aae9afac5c2c1bf7b27d187d10778a1ee", null ], + [ "docsis128_crc32_sec_ooo", "structIMB__MGR.html#acf438b50cb95a4fff1276b6666d22ad4", null ], + [ "docsis128_sec_ooo", "structIMB__MGR.html#a93cb5904608e179c49917f80c64fabd1", null ], + [ "docsis256_crc32_sec_ooo", "structIMB__MGR.html#af6531598b8dd0131a11e0cf95854605d", null ], + [ "docsis256_sec_ooo", "structIMB__MGR.html#acb6e3b70e16194274e2eaefdff8f9e89", null ], + [ "docsis_des_dec_ooo", "structIMB__MGR.html#a3d5173cea3d4932576f0a5b3abbc1f40", null ], + [ "docsis_des_enc_ooo", "structIMB__MGR.html#a937fb0d3b113d203ee1a4beaf5b5b98f", null ], + [ "earliest_job", "structIMB__MGR.html#ac5cd33ac3bf985dc31ccc716fb3a7747", null ], + [ "eea3_1_buffer", "structIMB__MGR.html#aeff54b370fbe495ce96370811d280140", null ], + [ "eea3_4_buffer", "structIMB__MGR.html#ac5ad1bf8ecdcfc51eaf681fb353c9621", null ], + [ "eea3_n_buffer", "structIMB__MGR.html#a67d8a55ba87742b8cf282d23ed9e3ef5", null ], + [ "eia3_1_buffer", "structIMB__MGR.html#a3ed58bdf37ab6d5f0d28271882d52327", null ], + [ "eia3_n_buffer", "structIMB__MGR.html#ab515c5aec39b1a9a31454b945e5ccd35", null ], + [ "end_ooo", "structIMB__MGR.html#a6e1ab772182b97f931af3d4685785353", null ], + [ "f8_1_buffer", "structIMB__MGR.html#af854e664cef84cb8be4495a1af5e9cb5", null ], + [ "f8_1_buffer_bit", "structIMB__MGR.html#aacf6ce3f508fef115dcd2dc415ca7379", null ], + [ "f8_2_buffer", "structIMB__MGR.html#ac733fc4c5ee6e80e8e71b20a9bb2dc6e", null ], + [ "f8_3_buffer", "structIMB__MGR.html#a929b9c50bc98bf37daec7a1603f10118", null ], + [ "f8_4_buffer", "structIMB__MGR.html#af0b1c5d3b269b252a75a55df8bdf472c", null ], + [ "f8_n_buffer", "structIMB__MGR.html#ae0f2316d96e46b64d08e3cf73a93b39d", null ], + [ "f9_1_buffer", "structIMB__MGR.html#afafb95a00fe6633fb6901855928b08bb", null ], + [ "f9_1_buffer_user", "structIMB__MGR.html#a4fbdac066c9d0fbd1d586a2e5b2319c3", null ], + [ "features", "structIMB__MGR.html#af877c7669265775604857c6b81063bdb", null ], + [ "flags", "structIMB__MGR.html#ae1a596c9c9d6ca1292c6116575021c14", null ], + [ "flush_burst", "structIMB__MGR.html#aa0283df1a5cd7a6dfc78c8f9896ea3ec", null ], + [ "flush_job", "structIMB__MGR.html#a225e3b35eef8e18fd09c429fbb2f70b5", null ], + [ "gcm128_dec", "structIMB__MGR.html#a2ce2f6e6d61953b24a5859279534b02a", null ], + [ "gcm128_dec_finalize", "structIMB__MGR.html#af95fa831fb24629e19613e8594e5d32a", null ], + [ "gcm128_dec_update", "structIMB__MGR.html#a28ddfed1a807fedb9c77c99736278dcb", null ], + [ "gcm128_enc", "structIMB__MGR.html#a9f8b9687fbb17f7a78fc4ba79e2749fd", null ], + [ "gcm128_enc_finalize", "structIMB__MGR.html#a8efc0cf95907805289a297c024ce0cac", null ], + [ "gcm128_enc_update", "structIMB__MGR.html#ad2c64d9d5f9ffd65d13e73205cb803cd", null ], + [ "gcm128_init", "structIMB__MGR.html#ad8c004899d29974a3f4c034370422c31", null ], + [ "gcm128_init_var_iv", "structIMB__MGR.html#aebd9eab76cd056add559e6bea328dbf0", null ], + [ "gcm128_pre", "structIMB__MGR.html#a207c78e01e4aa1a35de1bbd2cc0a6ee2", null ], + [ "gcm128_precomp", "structIMB__MGR.html#a69e4dd044703cd3aa9c4c36e04b2a701", null ], + [ "gcm192_dec", "structIMB__MGR.html#adff2cf46e7cbb5e2926f348623d8ad2a", null ], + [ "gcm192_dec_finalize", "structIMB__MGR.html#a4d7879172344be144b92be68b2e2a81a", null ], + [ "gcm192_dec_update", "structIMB__MGR.html#a66fa94f9c07061f3d93b78ed505931e3", null ], + [ "gcm192_enc", "structIMB__MGR.html#aaaeb1676e0cf19e3558d10e537f71bda", null ], + [ "gcm192_enc_finalize", "structIMB__MGR.html#a91ad9ce7a4b3c9ceeffa9dd712013915", null ], + [ "gcm192_enc_update", "structIMB__MGR.html#a3b5c3407cc4595420dbe43e98090e558", null ], + [ "gcm192_init", "structIMB__MGR.html#a006315a7045996d451c7dfd03b96a076", null ], + [ "gcm192_init_var_iv", "structIMB__MGR.html#a7bfa7b17c0fb36dc832138a86948991f", null ], + [ "gcm192_pre", "structIMB__MGR.html#a149b463ecf68ee78d39c33e183ed8997", null ], + [ "gcm192_precomp", "structIMB__MGR.html#abec8bdb319ca132028d5b787bf60a895", null ], + [ "gcm256_dec", "structIMB__MGR.html#acfdfb511771cac71e032c5bf1c9273b7", null ], + [ "gcm256_dec_finalize", "structIMB__MGR.html#a5c2e4a1296e33f5c9eaff2067a6432d4", null ], + [ "gcm256_dec_update", "structIMB__MGR.html#a99fcb1d18fa45044063c6c288b0fc769", null ], + [ "gcm256_enc", "structIMB__MGR.html#a79767326692a8d4faabe5ab435dc5df8", null ], + [ "gcm256_enc_finalize", "structIMB__MGR.html#a9764d27efef2491a86d517f38dd9b6ef", null ], + [ "gcm256_enc_update", "structIMB__MGR.html#ac131ceec0dcb0d08097b76c8abe9e024", null ], + [ "gcm256_init", "structIMB__MGR.html#a8987d7d491628d9d80f12820468ac541", null ], + [ "gcm256_init_var_iv", "structIMB__MGR.html#afa275308dab467d0c68dc921bd68d1a4", null ], + [ "gcm256_pre", "structIMB__MGR.html#a44b0b9d218043be965a84c29ced2584b", null ], + [ "gcm256_precomp", "structIMB__MGR.html#a1ea15abc11b618ceb438c7f282643c52", null ], + [ "get_completed_job", "structIMB__MGR.html#a99b7111c8fc59161e0bf291d13b1f195", null ], + [ "get_next_burst", "structIMB__MGR.html#a8c6b03b886ce7706a6cc8319345e8acf", null ], + [ "get_next_job", "structIMB__MGR.html#a35939892cd168eb7ce9c6c29c5401709", null ], + [ "ghash", "structIMB__MGR.html#afc7d7ae10e8e6f37b9ec894059818fad", null ], + [ "ghash_pre", "structIMB__MGR.html#af6a1c323981b8c3cfe09f94cd9dd58e2", null ], + [ "gmac128_finalize", "structIMB__MGR.html#a00ce9a8422a961166ce9855d4815cc2e", null ], + [ "gmac128_init", "structIMB__MGR.html#a1098c38784f6c6e4e3a3380f8be8ecc1", null ], + [ "gmac128_update", "structIMB__MGR.html#a90aad4f47448a1959b7cf747438d7ce9", null ], + [ "gmac192_finalize", "structIMB__MGR.html#a000b24d7fbb9b57ed6bd4147d225e015", null ], + [ "gmac192_init", "structIMB__MGR.html#a3dd58a9d9b4fb6cf85fa26175ffac9a6", null ], + [ "gmac192_update", "structIMB__MGR.html#a5220f0d243291ae5a2f43fbc93d18725", null ], + [ "gmac256_finalize", "structIMB__MGR.html#a3bb98c9a9c8545f363b74627870c7038", null ], + [ "gmac256_init", "structIMB__MGR.html#aca7d6388863eb76b60651e7b16c41b82", null ], + [ "gmac256_update", "structIMB__MGR.html#a0720a1fc9c0652c6b3aa7dfab070f73d", null ], + [ "hec_32", "structIMB__MGR.html#a0863c9f64126aba5cb47628493dae582", null ], + [ "hec_64", "structIMB__MGR.html#a4128ee02f4cbef5aa1e6b0bb5fd46097", null ], + [ "hmac_md5_ooo", "structIMB__MGR.html#a855d14df536b22ceb175dc495ced891c", null ], + [ "hmac_sha_1_ooo", "structIMB__MGR.html#a49be1d77b96d338c08200dce82e452d3", null ], + [ "hmac_sha_224_ooo", "structIMB__MGR.html#ab9226cf37e41c2a13ae6003305654193", null ], + [ "hmac_sha_256_ooo", "structIMB__MGR.html#a633807f7eaae35ada650a2c22f1366be", null ], + [ "hmac_sha_384_ooo", "structIMB__MGR.html#afd72d6af1d8ff605e8aaaf984f0700ed", null ], + [ "hmac_sha_512_ooo", "structIMB__MGR.html#af0cf9149495b0805a4f6f511c0fbff5b", null ], + [ "imb_errno", "structIMB__MGR.html#a0ab7a3dfdfd8442f591699e2310df11f", null ], + [ "jobs", "structIMB__MGR.html#ab342224dce812fb13a9e935e7a7b0617", null ], + [ "kasumi_init_f8_key_sched", "structIMB__MGR.html#ab32d67f27cb6f06980080e09db133b24", null ], + [ "kasumi_init_f9_key_sched", "structIMB__MGR.html#a58ad87104b08ba5b56fc65928677a639", null ], + [ "kasumi_key_sched_size", "structIMB__MGR.html#abd0c855a3e43df48f5f21686a095f358", null ], + [ "keyexp_128", "structIMB__MGR.html#a7c97c5361b90abe325db55530b6f198e", null ], + [ "keyexp_192", "structIMB__MGR.html#a2bd7444a318de06b5f53178abf9c37ff", null ], + [ "keyexp_256", "structIMB__MGR.html#a431e99ddb3837a32bedb6cd426054bcc", null ], + [ "md5_one_block", "structIMB__MGR.html#a54b5f7b625ce226ebd46a2a19e9bc724", null ], + [ "next_job", "structIMB__MGR.html#a106ebd60d876e56bfc6bcc03d20acce4", null ], + [ "queue_size", "structIMB__MGR.html#a5a35c9c2a1c8eb5b4aa239a3b04f4fbf", null ], + [ "reserved", "structIMB__MGR.html#a136697eb37d5a06eeada0f338abd8b51", null ], + [ "set_suite_id", "structIMB__MGR.html#a95209e74a7acff69d46eef91021a1168", null ], + [ "sha1", "structIMB__MGR.html#a82aaedc800096aaa55062c56131e0b13", null ], + [ "sha1_one_block", "structIMB__MGR.html#a80a26be9a12f561a68b70d65e096cde9", null ], + [ "sha224", "structIMB__MGR.html#a52e26b8805b28d1298ac48e5762412b5", null ], + [ "sha224_one_block", "structIMB__MGR.html#aacf8209094d12ceceed96d4d0fae193c", null ], + [ "sha256", "structIMB__MGR.html#a8d978764a953e220b3c799990bae490b", null ], + [ "sha256_one_block", "structIMB__MGR.html#ade4a72c39980f0bba790b70529b60158", null ], + [ "sha384", "structIMB__MGR.html#a9ffcce0ba9aef3cd68bdd7f6e9e3e2a0", null ], + [ "sha384_one_block", "structIMB__MGR.html#a92b7ff145f0016dd46eabf9271ef03f1", null ], + [ "sha512", "structIMB__MGR.html#a62ebdd5e2b5a59c9e648511caf56f3f1", null ], + [ "sha512_one_block", "structIMB__MGR.html#a88f1060a2adb8098fc085fc2b812addc", null ], + [ "sha_1_ooo", "structIMB__MGR.html#acdb53e812f3e4a307a497eeab28c4256", null ], + [ "sha_224_ooo", "structIMB__MGR.html#aa174bf6846067a48f3f47ee5ebd74877", null ], + [ "sha_256_ooo", "structIMB__MGR.html#a37fc5ccae7af099a8517566a20751265", null ], + [ "sha_384_ooo", "structIMB__MGR.html#a51a499114f84eed88cb48bd3b461bd1c", null ], + [ "sha_512_ooo", "structIMB__MGR.html#a6e21e034cd011b695bac5a711be65da1", null ], + [ "snow3g_f8_1_buffer", "structIMB__MGR.html#ab70cbec09a61b5adeae37260c10234ce", null ], + [ "snow3g_f8_1_buffer_bit", "structIMB__MGR.html#ad85fdcb101391b0a2488a3244d5f442d", null ], + [ "snow3g_f8_2_buffer", "structIMB__MGR.html#a25f46509b2f8926902a73e2210d697fd", null ], + [ "snow3g_f8_4_buffer", "structIMB__MGR.html#a194a0f15a7345ee369780dbdeb4f52f6", null ], + [ "snow3g_f8_8_buffer", "structIMB__MGR.html#a620d2d75bb50784aa0000b458197afaf", null ], + [ "snow3g_f8_8_buffer_multikey", "structIMB__MGR.html#a898a4c56e1a746a328529c3665c5682e", null ], + [ "snow3g_f8_n_buffer", "structIMB__MGR.html#a829c4cf1157dfc3fcb829df31e408e9d", null ], + [ "snow3g_f8_n_buffer_multikey", "structIMB__MGR.html#a63a2a5f74e7cc2fc637896bdaeb6b262", null ], + [ "snow3g_f9_1_buffer", "structIMB__MGR.html#a9a4d2148b6e8b4426b2946e9fe695461", null ], + [ "snow3g_init_key_sched", "structIMB__MGR.html#a38ff0528815be4d92481948664366ca9", null ], + [ "snow3g_key_sched_size", "structIMB__MGR.html#a5e0002330399f643e8f0ca0c1b96a7a0", null ], + [ "snow3g_uea2_ooo", "structIMB__MGR.html#ab0193bf46978026a82803c25e11acbfb", null ], + [ "snow3g_uia2_ooo", "structIMB__MGR.html#aa7d4cfe53e0d70c22d3a274b8cd218af", null ], + [ "submit_burst", "structIMB__MGR.html#aa69002f8d8e17c0048d7aff9b3a1894e", null ], + [ "submit_burst_nocheck", "structIMB__MGR.html#a34adc55c8041e211c73c632628cd60d3", null ], + [ "submit_cipher_burst", "structIMB__MGR.html#a8b52f1b3ca92eff48b945929a0eab839", null ], + [ "submit_cipher_burst_nocheck", "structIMB__MGR.html#acb2ee84d30247bb257635063db3ae308", null ], + [ "submit_hash_burst", "structIMB__MGR.html#a20a3d4d1e40aed26d83db5395b7ccc83", null ], + [ "submit_hash_burst_nocheck", "structIMB__MGR.html#a809782eb9dcbd264442bcd9edd2f4c2f", null ], + [ "submit_job", "structIMB__MGR.html#ab143534d96e0d585b19dfc19553e929b", null ], + [ "submit_job_nocheck", "structIMB__MGR.html#aab7350eeecc56b1ef45606b83519a72c", null ], + [ "used_arch", "structIMB__MGR.html#a44b6c95f8ce611828b6acacdb459cdc9", null ], + [ "xcbc_keyexp", "structIMB__MGR.html#ae575c2350f5fca9a182c541795bb7d4d", null ], + [ "zuc256_eea3_ooo", "structIMB__MGR.html#a6d05b0a20196be514182920710dce2a8", null ], + [ "zuc256_eia3_ooo", "structIMB__MGR.html#a5f8744a8d64273d3f4cbcd9ad41ce3d5", null ], + [ "zuc_eea3_ooo", "structIMB__MGR.html#a26d22c5ff963b9c7ffc8af19292bce5c", null ], + [ "zuc_eia3_ooo", "structIMB__MGR.html#a910f747aa1a0e6a46f87b0dfd2189219", null ] +]; \ No newline at end of file diff --git a/docs/structIMB__SGL__IOV.html b/docs/structIMB__SGL__IOV.html new file mode 100644 index 0000000000000000000000000000000000000000..3c4169b3b911fb6e747b019b738ce94ab4fb58ff --- /dev/null +++ b/docs/structIMB__SGL__IOV.html @@ -0,0 +1,176 @@ + + + + + + + +intel-ipsec-mb: IMB_SGL_IOV Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
IMB_SGL_IOV Struct Reference
+
+
+ +

#include <intel-ipsec-mb.h>

+ + + + + + + + +

+Data Fields

const void * in
 
void * out
 
uint64_t len
 
+

Detailed Description

+

Input/output SGL segment structure.

+

Field Documentation

+ +

◆ in

+ +
+
+ + + + +
const void* IMB_SGL_IOV::in
+
+

Input segment

+ +
+
+ +

◆ len

+ +
+
+ + + + +
uint64_t IMB_SGL_IOV::len
+
+ +
+
+ +

◆ out

+ +
+
+ + + + +
void* IMB_SGL_IOV::out
+
+

Output segment

+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structchacha20__poly1305__context__data.html b/docs/structchacha20__poly1305__context__data.html new file mode 100644 index 0000000000000000000000000000000000000000..207904590284de88ba31d045ebe3feea8fab3f1f --- /dev/null +++ b/docs/structchacha20__poly1305__context__data.html @@ -0,0 +1,299 @@ + + + + + + + +intel-ipsec-mb: chacha20_poly1305_context_data Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
chacha20_poly1305_context_data Struct Reference
+
+
+ +

holds Chacha20-Poly1305 operation context + More...

+ +

#include <intel-ipsec-mb.h>

+ + + + + + + + + + + + + + + + + + + + + + +

+Data Fields

uint64_t hash [3]
 
uint64_t aad_len
 
uint64_t hash_len
 
uint8_t last_ks [64]
 
uint8_t poly_key [32]
 
uint8_t poly_scratch [16]
 
uint64_t last_block_count
 
uint64_t remain_ks_bytes
 
uint64_t remain_ct_bytes
 
uint8_t IV [12]
 
+

Detailed Description

+

holds Chacha20-Poly1305 operation context

+

Field Documentation

+ +

◆ aad_len

+ +
+
+ + + + +
uint64_t chacha20_poly1305_context_data::aad_len
+
+

Total AAD length

+ +
+
+ +

◆ hash

+ +
+
+ + + + +
uint64_t chacha20_poly1305_context_data::hash[3]
+
+

Intermediate computation of hash value

+ +
+
+ +

◆ hash_len

+ +
+
+ + + + +
uint64_t chacha20_poly1305_context_data::hash_len
+
+

Total length to digest (excluding AAD)

+ +
+
+ +

◆ IV

+ +
+
+ + + + +
uint8_t chacha20_poly1305_context_data::IV[12]
+
+

IV (12 bytes)

+ +
+
+ +

◆ last_block_count

+ +
+
+ + + + +
uint64_t chacha20_poly1305_context_data::last_block_count
+
+

Last block count used in last segment

+ +
+
+ +

◆ last_ks

+ +
+
+ + + + +
uint8_t chacha20_poly1305_context_data::last_ks[64]
+
+

Last 64 bytes of KS

+ +
+
+ +

◆ poly_key

+ +
+
+ + + + +
uint8_t chacha20_poly1305_context_data::poly_key[32]
+
+

Poly key

+ +
+
+ +

◆ poly_scratch

+ +
+
+ + + + +
uint8_t chacha20_poly1305_context_data::poly_scratch[16]
+
+

Scratchpad to compute Poly on 16 bytes

+ +
+
+ +

◆ remain_ct_bytes

+ +
+
+ + + + +
uint64_t chacha20_poly1305_context_data::remain_ct_bytes
+
+

Amount of ciphertext bytes still to use of previous segment to authenticate (up to 16 bytes)

+ +
+
+ +

◆ remain_ks_bytes

+ +
+
+ + + + +
uint64_t chacha20_poly1305_context_data::remain_ks_bytes
+
+

Amount of bytes still to use of keystream (up to 63 bytes)

+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structgcm__context__data.html b/docs/structgcm__context__data.html new file mode 100644 index 0000000000000000000000000000000000000000..ad5f51f5ea1d61a4edf79fa10a9863e2bf789f39 --- /dev/null +++ b/docs/structgcm__context__data.html @@ -0,0 +1,242 @@ + + + + + + + +intel-ipsec-mb: gcm_context_data Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
gcm_context_data Struct Reference
+
+
+ +

holds GCM operation context + More...

+ +

#include <intel-ipsec-mb.h>

+ + + + + + + + + + + + + + + + +

+Data Fields

uint8_t aad_hash [IMB_GCM_BLOCK_LEN]
 
uint64_t aad_length
 
uint64_t in_length
 
uint8_t partial_block_enc_key [IMB_GCM_BLOCK_LEN]
 
uint8_t orig_IV [IMB_GCM_BLOCK_LEN]
 
uint8_t current_counter [IMB_GCM_BLOCK_LEN]
 
uint64_t partial_block_length
 
+

Detailed Description

+

holds GCM operation context

+

init, update and finalize context data

+

Field Documentation

+ +

◆ aad_hash

+ +
+
+ + + + +
uint8_t gcm_context_data::aad_hash[IMB_GCM_BLOCK_LEN]
+
+ +
+
+ +

◆ aad_length

+ +
+
+ + + + +
uint64_t gcm_context_data::aad_length
+
+ +
+
+ +

◆ current_counter

+ +
+
+ + + + +
uint8_t gcm_context_data::current_counter[IMB_GCM_BLOCK_LEN]
+
+ +
+
+ +

◆ in_length

+ +
+
+ + + + +
uint64_t gcm_context_data::in_length
+
+ +
+
+ +

◆ orig_IV

+ +
+
+ + + + +
uint8_t gcm_context_data::orig_IV[IMB_GCM_BLOCK_LEN]
+
+ +
+
+ +

◆ partial_block_enc_key

+ +
+
+ + + + +
uint8_t gcm_context_data::partial_block_enc_key[IMB_GCM_BLOCK_LEN]
+
+ +
+
+ +

◆ partial_block_length

+ +
+
+ + + + +
uint64_t gcm_context_data::partial_block_length
+
+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structgcm__key__data.html b/docs/structgcm__key__data.html new file mode 100644 index 0000000000000000000000000000000000000000..ae5aa201e2628c374c1ea1185ca0c84132cc145e --- /dev/null +++ b/docs/structgcm__key__data.html @@ -0,0 +1,260 @@ + + + + + + + +intel-ipsec-mb: gcm_key_data Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
gcm_key_data Struct Reference
+
+
+ +

holds intermediate key data needed to improve performance + More...

+ +

#include <intel-ipsec-mb.h>

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Data Fields

uint8_t expanded_keys [IMB_GCM_ENC_KEY_LEN *IMB_GCM_KEY_SETS]
 
union { 
 
   struct { 
 
      uint8_t   shifted_hkey [IMB_GCM_ENC_KEY_LEN *8] 
 
      uint8_t   shifted_hkey_k [IMB_GCM_ENC_KEY_LEN *8] 
 
   }   sse_avx 
 
   struct { 
 
      uint8_t   shifted_hkey [IMB_GCM_ENC_KEY_LEN *8] 
 
   }   avx2_avx512 
 
   struct { 
 
      uint8_t   shifted_hkey [IMB_GCM_ENC_KEY_LEN *32 *2] 
 
   }   vaes_avx512 
 
} ghash_keys 
 
+

Detailed Description

+

holds intermediate key data needed to improve performance

+

gcm_key_data hold internal key information used by gcm128, gcm192 and gcm256.

+

Field Documentation

+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } gcm_key_data::avx2_avx512
+
+ +
+
+ +

◆ expanded_keys

+ +
+
+ + + + +
uint8_t gcm_key_data::expanded_keys[IMB_GCM_ENC_KEY_LEN *IMB_GCM_KEY_SETS]
+
+ +
+
+ +

◆ [union]

+ +
+
+ + + + +
union { ... } gcm_key_data::ghash_keys
+
+ +
+
+ +

◆ shifted_hkey

+ +
+
+ + + + +
uint8_t gcm_key_data::shifted_hkey[IMB_GCM_ENC_KEY_LEN *32 *2]
+
+

This is needed for schoolbook multiply purposes. (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., (Hashkey^48<<1 mod poly)

+

Start with: (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., (Hashkey^32<<1 mod poly)

+

Followed by: (HashKey<<1 mod poly) x POLY, (HashKey^2<<1 mod poly) x POLY, ..., (Hashkey^32<<1 mod poly) x POLY

+ +
+
+ +

◆ shifted_hkey_k

+ +
+
+ + + + +
uint8_t gcm_key_data::shifted_hkey_k[IMB_GCM_ENC_KEY_LEN *8]
+
+

This is needed for Karatsuba multiply purposes. Storage for XOR of High 64 bits and low 64 bits of HashKey mod poly.

+

(HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., (Hashkey^128<<1 mod poly)

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } gcm_key_data::sse_avx
+
+

< Storage for precomputed hash keys

+ +
+
+ +

◆ [struct]

+ +
+
+ + + + +
struct { ... } gcm_key_data::vaes_avx512
+
+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structgcm__key__data.js b/docs/structgcm__key__data.js new file mode 100644 index 0000000000000000000000000000000000000000..77d95b910e218da93ffffcf3118f99c6b1368a4a --- /dev/null +++ b/docs/structgcm__key__data.js @@ -0,0 +1,10 @@ +var structgcm__key__data = +[ + [ "avx2_avx512", "structgcm__key__data.html#a3f231fdc1d521cd15c111667e4aea818", null ], + [ "expanded_keys", "structgcm__key__data.html#aa5bda4f55b8393be151cfe28a9dc1ca9", null ], + [ "ghash_keys", "structgcm__key__data.html#af5adb4225d569050207cfcfe00fe273a", null ], + [ "shifted_hkey", "structgcm__key__data.html#a1c73e3891c50c5e03e82715ceda77d17", null ], + [ "shifted_hkey_k", "structgcm__key__data.html#a570a82f076ceb5da6719d1a554584820", null ], + [ "sse_avx", "structgcm__key__data.html#a908988e25607cd3e092bfcea9964deb1", null ], + [ "vaes_avx512", "structgcm__key__data.html#a7ad44e105b23e41b7f79530b6370e509", null ] +]; \ No newline at end of file diff --git a/docs/structimb__uint128__t.html b/docs/structimb__uint128__t.html new file mode 100644 index 0000000000000000000000000000000000000000..e9aeb5928c51883f7a8f0b839ac50b62e8320819 --- /dev/null +++ b/docs/structimb__uint128__t.html @@ -0,0 +1,156 @@ + + + + + + + +intel-ipsec-mb: imb_uint128_t Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
imb_uint128_t Struct Reference
+
+
+ +

#include <intel-ipsec-mb.h>

+ + + + + + +

+Data Fields

uint64_t low
 
uint64_t high
 
+

Field Documentation

+ +

◆ high

+ +
+
+ + + + +
uint64_t imb_uint128_t::high
+
+ +
+
+ +

◆ low

+ +
+
+ + + + +
uint64_t imb_uint128_t::low
+
+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structkasumi__key__sched__s.html b/docs/structkasumi__key__sched__s.html new file mode 100644 index 0000000000000000000000000000000000000000..f9d2570475ffd253a548f20dc706c975c0680c2a --- /dev/null +++ b/docs/structkasumi__key__sched__s.html @@ -0,0 +1,160 @@ + + + + + + + +intel-ipsec-mb: kasumi_key_sched_s Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
kasumi_key_sched_s Struct Reference
+
+
+ +

#include <intel-ipsec-mb.h>

+ + + + + + +

+Data Fields

uint16_t sk16 [KASUMI_KEY_SCHEDULE_SIZE]
 
uint16_t msk16 [KASUMI_KEY_SCHEDULE_SIZE]
 
+

Detailed Description

+

Structure to maintain internal key scheduling

+

Field Documentation

+ +

◆ msk16

+ +
+
+ + + + +
uint16_t kasumi_key_sched_s::msk16[KASUMI_KEY_SCHEDULE_SIZE]
+
+

modified key schedule

+ +
+
+ +

◆ sk16

+ +
+
+ + + + +
uint16_t kasumi_key_sched_s::sk16[KASUMI_KEY_SCHEDULE_SIZE]
+
+

< Kasumi internal scheduling key schedule

+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/structsnow3g__key__schedule__s.html b/docs/structsnow3g__key__schedule__s.html new file mode 100644 index 0000000000000000000000000000000000000000..1db8bdbb1cd83d82363e8c81e699713479bbdb82 --- /dev/null +++ b/docs/structsnow3g__key__schedule__s.html @@ -0,0 +1,142 @@ + + + + + + + +intel-ipsec-mb: snow3g_key_schedule_s Struct Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb v1.4 +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ +   + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Data Fields
+
snow3g_key_schedule_s Struct Reference
+
+
+ +

#include <intel-ipsec-mb.h>

+ + + + +

+Data Fields

uint32_t k [4]
 
+

Detailed Description

+

Snow3G key scheduling structure

+

Field Documentation

+ +

◆ k

+ +
+
+ + + + +
uint32_t snow3g_key_schedule_s::k[4]
+
+ +
+
+
The documentation for this struct was generated from the following file: +
+
+ + + + diff --git a/docs/tab_ad.png b/docs/tab_ad.png new file mode 100644 index 0000000000000000000000000000000000000000..e34850acfc24be58da6d2fd1ccc6b29cc84fe34d Binary files /dev/null and b/docs/tab_ad.png differ diff --git a/docs/tab_bd.png b/docs/tab_bd.png new file mode 100644 index 0000000000000000000000000000000000000000..91c25249869ffe5269c24f8c441e3f1bf75df400 Binary files /dev/null and b/docs/tab_bd.png differ diff --git a/docs/tab_hd.png b/docs/tab_hd.png new file mode 100644 index 0000000000000000000000000000000000000000..2489273d4ce1d11d3f7557335244c02b9959351b Binary files /dev/null and b/docs/tab_hd.png differ diff --git a/docs/tab_sd.png b/docs/tab_sd.png new file mode 100644 index 0000000000000000000000000000000000000000..757a565ced4730f85c833fb2547d8e199ae68f19 Binary files /dev/null and b/docs/tab_sd.png differ diff --git a/docs/tabs.css b/docs/tabs.css new file mode 100644 index 0000000000000000000000000000000000000000..71c8a4704c0e4248120fe7c8e910d4dbbea21841 --- /dev/null +++ b/docs/tabs.css @@ -0,0 +1 @@ +.sm{position:relative;z-index:9999}.sm,.sm ul,.sm li{display:block;list-style:none;margin:0;padding:0;line-height:normal;direction:ltr;text-align:left;-webkit-tap-highlight-color:rgba(0,0,0,0)}.sm-rtl,.sm-rtl ul,.sm-rtl li{direction:rtl;text-align:right}.sm>li>h1,.sm>li>h2,.sm>li>h3,.sm>li>h4,.sm>li>h5,.sm>li>h6{margin:0;padding:0}.sm ul{display:none}.sm li,.sm a{position:relative}.sm a{display:block}.sm a.disabled{cursor:not-allowed}.sm:after{content:"\00a0";display:block;height:0;font:0/0 serif;clear:both;visibility:hidden;overflow:hidden}.sm,.sm *,.sm *:before,.sm *:after{-moz-box-sizing:border-box;-webkit-box-sizing:border-box;box-sizing:border-box}.main-menu-btn{position:relative;display:inline-block;width:36px;height:36px;text-indent:36px;margin-left:8px;white-space:nowrap;overflow:hidden;cursor:pointer;-webkit-tap-highlight-color:rgba(0,0,0,0)}.main-menu-btn-icon,.main-menu-btn-icon:before,.main-menu-btn-icon:after{position:absolute;top:50%;left:2px;height:2px;width:24px;background:var(--nav-menu-button-color);-webkit-transition:all .25s;transition:all .25s}.main-menu-btn-icon:before{content:'';top:-7px;left:0}.main-menu-btn-icon:after{content:'';top:7px;left:0}#main-menu-state:checked ~ .main-menu-btn .main-menu-btn-icon{height:0}#main-menu-state:checked ~ .main-menu-btn .main-menu-btn-icon:before{top:0;-webkit-transform:rotate(-45deg);transform:rotate(-45deg)}#main-menu-state:checked ~ .main-menu-btn .main-menu-btn-icon:after{top:0;-webkit-transform:rotate(45deg);transform:rotate(45deg)}#main-menu-state{position:absolute;width:1px;height:1px;margin:-1px;border:0;padding:0;overflow:hidden;clip:rect(1px,1px,1px,1px)}#main-menu-state:not(:checked) ~ #main-menu{display:none}#main-menu-state:checked ~ #main-menu{display:block}@media(min-width:768px){.main-menu-btn{position:absolute;top:-99999px}#main-menu-state:not(:checked) ~ #main-menu{display:block}}.sm-dox{background-image:var(--nav-gradient-image)}.sm-dox a,.sm-dox a:focus,.sm-dox a:hover,.sm-dox a:active{padding:0 12px;padding-right:43px;font-family:var(--font-family-nav);font-size:13px;font-weight:bold;line-height:36px;text-decoration:none;text-shadow:var(--nav-text-normal-shadow);color:var(--nav-text-normal-color);outline:0}.sm-dox a:hover{background-image:var(--nav-gradient-active-image);background-repeat:repeat-x;color:var(--nav-text-hover-color);text-shadow:var(--nav-text-hover-shadow)}.sm-dox a.current{color:#d23600}.sm-dox a.disabled{color:#bbb}.sm-dox a span.sub-arrow{position:absolute;top:50%;margin-top:-14px;left:auto;right:3px;width:28px;height:28px;overflow:hidden;font:bold 12px/28px monospace !important;text-align:center;text-shadow:none;background:var(--nav-menu-toggle-color);-moz-border-radius:5px;-webkit-border-radius:5px;border-radius:5px}.sm-dox a span.sub-arrow:before{display:block;content:'+'}.sm-dox a.highlighted span.sub-arrow:before{display:block;content:'-'}.sm-dox>li:first-child>a,.sm-dox>li:first-child>:not(ul) a{-moz-border-radius:5px 5px 0 0;-webkit-border-radius:5px;border-radius:5px 5px 0 0}.sm-dox>li:last-child>a,.sm-dox>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul{-moz-border-radius:0 0 5px 5px;-webkit-border-radius:0;border-radius:0 0 5px 5px}.sm-dox>li:last-child>a.highlighted,.sm-dox>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a.highlighted{-moz-border-radius:0;-webkit-border-radius:0;border-radius:0}.sm-dox ul{background:var(--nav-menu-background-color)}.sm-dox ul a,.sm-dox ul a:focus,.sm-dox ul a:hover,.sm-dox ul a:active{font-size:12px;border-left:8px solid transparent;line-height:36px;text-shadow:none;background-color:var(--nav-menu-background-color);background-image:none}.sm-dox ul a:hover{background-image:var(--nav-gradient-active-image);background-repeat:repeat-x;color:var(--nav-text-hover-color);text-shadow:0 1px 1px black}.sm-dox ul ul a,.sm-dox ul ul a:hover,.sm-dox ul ul a:focus,.sm-dox ul ul a:active{border-left:16px solid transparent}.sm-dox ul ul ul a,.sm-dox ul ul ul a:hover,.sm-dox ul ul ul a:focus,.sm-dox ul ul ul a:active{border-left:24px solid transparent}.sm-dox ul ul ul ul a,.sm-dox ul ul ul ul a:hover,.sm-dox ul ul ul ul a:focus,.sm-dox ul ul ul ul a:active{border-left:32px solid transparent}.sm-dox ul ul ul ul ul a,.sm-dox ul ul ul ul ul a:hover,.sm-dox ul ul ul ul ul a:focus,.sm-dox ul ul ul ul ul a:active{border-left:40px solid transparent}@media(min-width:768px){.sm-dox ul{position:absolute;width:12em}.sm-dox li{float:left}.sm-dox.sm-rtl li{float:right}.sm-dox ul li,.sm-dox.sm-rtl ul li,.sm-dox.sm-vertical li{float:none}.sm-dox a{white-space:nowrap}.sm-dox ul a,.sm-dox.sm-vertical a{white-space:normal}.sm-dox .sm-nowrap>li>a,.sm-dox .sm-nowrap>li>:not(ul) a{white-space:nowrap}.sm-dox{padding:0 10px;background-image:var(--nav-gradient-image);line-height:36px}.sm-dox a span.sub-arrow{top:50%;margin-top:-2px;right:12px;width:0;height:0;border-width:4px;border-style:solid dashed dashed dashed;border-color:var(--nav-text-normal-color) transparent transparent transparent;background:transparent;-moz-border-radius:0;-webkit-border-radius:0;border-radius:0}.sm-dox a,.sm-dox a:focus,.sm-dox a:active,.sm-dox a:hover,.sm-dox a.highlighted{padding:0 12px;background-image:var(--nav-separator-image);background-repeat:no-repeat;background-position:right;-moz-border-radius:0 !important;-webkit-border-radius:0;border-radius:0 !important}.sm-dox a:hover{background-image:var(--nav-gradient-active-image);background-repeat:repeat-x;color:var(--nav-text-hover-color);text-shadow:var(--nav-text-hover-shadow)}.sm-dox a:hover span.sub-arrow{border-color:var(--nav-text-hover-color) transparent transparent transparent}.sm-dox a.has-submenu{padding-right:24px}.sm-dox li{border-top:0}.sm-dox>li>ul:before,.sm-dox>li>ul:after{content:'';position:absolute;top:-18px;left:30px;width:0;height:0;overflow:hidden;border-width:9px;border-style:dashed dashed solid dashed;border-color:transparent transparent #bbb transparent}.sm-dox>li>ul:after{top:-16px;left:31px;border-width:8px;border-color:transparent transparent var(--nav-menu-background-color) transparent}.sm-dox ul{border:1px solid #bbb;padding:5px 0;background:var(--nav-menu-background-color);-moz-border-radius:5px !important;-webkit-border-radius:5px;border-radius:5px !important;-moz-box-shadow:0 5px 9px rgba(0,0,0,0.2);-webkit-box-shadow:0 5px 9px rgba(0,0,0,0.2);box-shadow:0 5px 9px rgba(0,0,0,0.2)}.sm-dox ul a span.sub-arrow{right:8px;top:50%;margin-top:-5px;border-width:5px;border-color:transparent transparent transparent var(--nav-menu-foreground-color);border-style:dashed dashed dashed solid}.sm-dox ul a,.sm-dox ul a:hover,.sm-dox ul a:focus,.sm-dox ul a:active,.sm-dox ul a.highlighted{color:var(--nav-menu-foreground-color);background-image:none;border:0 !important;color:var(--nav-menu-foreground-color);background-image:none}.sm-dox ul a:hover{background-image:var(--nav-gradient-active-image);background-repeat:repeat-x;color:var(--nav-text-hover-color);text-shadow:var(--nav-text-hover-shadow)}.sm-dox ul a:hover span.sub-arrow{border-color:transparent transparent transparent var(--nav-text-hover-color)}.sm-dox span.scroll-up,.sm-dox span.scroll-down{position:absolute;display:none;visibility:hidden;overflow:hidden;background:var(--nav-menu-background-color);height:36px}.sm-dox span.scroll-up:hover,.sm-dox span.scroll-down:hover{background:#eee}.sm-dox span.scroll-up:hover span.scroll-up-arrow,.sm-dox span.scroll-up:hover span.scroll-down-arrow{border-color:transparent transparent #d23600 transparent}.sm-dox span.scroll-down:hover span.scroll-down-arrow{border-color:#d23600 transparent transparent transparent}.sm-dox span.scroll-up-arrow,.sm-dox span.scroll-down-arrow{position:absolute;top:0;left:50%;margin-left:-6px;width:0;height:0;overflow:hidden;border-width:6px;border-style:dashed dashed solid dashed;border-color:transparent transparent var(--nav-menu-foreground-color) transparent}.sm-dox span.scroll-down-arrow{top:8px;border-style:solid dashed dashed dashed;border-color:var(--nav-menu-foreground-color) transparent transparent transparent}.sm-dox.sm-rtl a.has-submenu{padding-right:12px;padding-left:24px}.sm-dox.sm-rtl a span.sub-arrow{right:auto;left:12px}.sm-dox.sm-rtl.sm-vertical a.has-submenu{padding:10px 20px}.sm-dox.sm-rtl.sm-vertical a span.sub-arrow{right:auto;left:8px;border-style:dashed solid dashed dashed;border-color:transparent #555 transparent transparent}.sm-dox.sm-rtl>li>ul:before{left:auto;right:30px}.sm-dox.sm-rtl>li>ul:after{left:auto;right:31px}.sm-dox.sm-rtl ul a.has-submenu{padding:10px 20px !important}.sm-dox.sm-rtl ul a span.sub-arrow{right:auto;left:8px;border-style:dashed solid dashed dashed;border-color:transparent #555 transparent transparent}.sm-dox.sm-vertical{padding:10px 0;-moz-border-radius:5px;-webkit-border-radius:5px;border-radius:5px}.sm-dox.sm-vertical a{padding:10px 20px}.sm-dox.sm-vertical a:hover,.sm-dox.sm-vertical a:focus,.sm-dox.sm-vertical a:active,.sm-dox.sm-vertical a.highlighted{background:#fff}.sm-dox.sm-vertical a.disabled{background-image:var(--nav-gradient-image)}.sm-dox.sm-vertical a span.sub-arrow{right:8px;top:50%;margin-top:-5px;border-width:5px;border-style:dashed dashed dashed solid;border-color:transparent transparent transparent #555}.sm-dox.sm-vertical>li>ul:before,.sm-dox.sm-vertical>li>ul:after{display:none}.sm-dox.sm-vertical ul a{padding:10px 20px}.sm-dox.sm-vertical ul a:hover,.sm-dox.sm-vertical ul a:focus,.sm-dox.sm-vertical ul a:active,.sm-dox.sm-vertical ul a.highlighted{background:#eee}.sm-dox.sm-vertical ul a.disabled{background:var(--nav-menu-background-color)}} \ No newline at end of file diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..65a2c999c39ef7fd630379987187797c2312233d --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,32 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +# build imb-burst application +add_subdirectory(burst-app) diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7eab0c8d1c63858bf907dd2145afea1df280071a --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,71 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +BURST_APP := burst-app + +# targets come here +all: $(BURST_APP) + +.PHONY: $(BURST_APP) + +$(BURST_APP): + $(MAKE) -C burst-app +.PHONY: clean +clean: + $(MAKE) -C burst-app clean + +# style check section +CHECKPATCH?=checkpatch.pl +CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --max-line-length=100 --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,AVOID_EXTERNS,COMPLEX_MACRO,USE_FUNC,CONSTANT_COMPARISON,MISSING_SPACE,NEW_TYPEDEFS + +%.c_style_check : %.c + $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< + +BURST_APP_SOURCES := $(wildcard $(BURST_APP)/*.[c]) +SOURCES_ALL := $(BURST_APP_SOURCES) +SOURCES_STYLE := $(foreach infile,$(SOURCES_ALL),$(infile)_style_check) + +.PHONY: style +style: $(SOURCES_STYLE) + +# cppcheck analysis check +CPPCHECK ?= cppcheck +CPPCHECK_OPTS ?= -I./ -I../lib +CPPCHECK_FLAGS ?= -j $(shell getconf _NPROCESSORS_ONLN) +CPPCHECK_FLAGS1 ?= --cppcheck-build-dir=.cppcheck $(CPPCHECK_FLAGS) +CPPCHECK_FLAGS2 ?= --cppcheck-build-dir=.bughunt $(CPPCHECK_FLAGS) +.PHONY: cppcheck +cppcheck: + mkdir -p .cppcheck + $(CPPCHECK) --force --enable=all $(CPPCHECK_FLAGS1) $(CPPCHECK_OPTS) \ + ./burst-app + +.PHONY: bughunt +bughunt: + mkdir -p .bughunt + $(CPPCHECK) --bug-hunting --inconclusive $(CPPCHECK_FLAGS2) $(CPPCHECK_OPTS) \ + ./burst-app diff --git a/examples/burst-app/CMakeLists.txt b/examples/burst-app/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..de391d813b95d020d1bca61d8cf0509bbf282faa --- /dev/null +++ b/examples/burst-app/CMakeLists.txt @@ -0,0 +1,65 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +######################################## +# set app and library names +######################################## +set(SAMPLE_APP imb-burst) +set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib) +set(TEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include) + +######################################## +# set imb-burst source files +######################################## +set(SAMPLE_APP_SRC_FILES_C + ${CMAKE_CURRENT_SOURCE_DIR}/main.c +) + +######################################## +# set C compiler options +######################################## +# add OS specific options +if(CMAKE_HOST_UNIX) + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/unix.cmake) +else() + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/windows.cmake) +endif() + + +######################################## +# add targets +######################################## +include_directories(${LIB_DIR}) +link_directories(${LIB_DIR}) + +add_executable(${SAMPLE_APP} ${SAMPLE_APP_SRC_FILES_C}) +target_link_libraries(${SAMPLE_APP} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${SAMPLE_APP} PRIVATE ${APP_DEFINES}) + diff --git a/examples/burst-app/Makefile b/examples/burst-app/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d441cb652b9c69d62f629ba159be47aeb755ac7b --- /dev/null +++ b/examples/burst-app/Makefile @@ -0,0 +1,73 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP := imb-burst +INSTPATH ?= /usr/include/ipsec-mb.h +LIB_DIR ?= ../../lib + +CFLAGS = -D_GNU_SOURCE -DNO_COMPAT_IMB_API_053 $(INCLUDES) \ + -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ + -Wcast-qual -Wundef -Wwrite-strings \ + -Wformat -Wformat-security \ + -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ + -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ + -pthread -fno-delete-null-pointer-checks -fwrapv -std=c99 + +# -fno-strict-overflow is not supported by clang +ifneq ($(CC),clang) +CFLAGS += -fno-strict-overflow +endif + +LDLIBS = -lIPSec_MB + +# imb-burst modules +SOURCES := main.c +OBJECTS := $(SOURCES:%.c=%.o) + +ifeq ("$(shell test -e $(INSTPATH) && echo -n yes)","yes") +# library installed +CFLAGS += +else +# library not installed +CFLAGS += -I$(LIB_DIR) +LDFLAGS += -L$(LIB_DIR) +endif + +# targets come here +all: $(APP) + +$(APP): $(OBJECTS) + $(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +.PHONY: clean +clean: + -rm -f $(DEP_FILES) $(OBJECTS) $(APP) + +# if target not clean then make dependencies +ifneq ($(MAKECMDGOALS),clean) +-include $(wildcard *.d) +endif diff --git a/examples/burst-app/README.md b/examples/burst-app/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88f32dc7456f6a1284178340c77b5cb6e17b2848 --- /dev/null +++ b/examples/burst-app/README.md @@ -0,0 +1,21 @@ +# Intel(R) Multi-Buffer Crypto for IPsec Library - Basic Sample Application + +## Contents + +- Overview +- Usage + + +## Overview + +The basic sample application showcases how to use the library API, +encrypting several buffers with AES-128-GCM using the burst API, +with all steps required, minimizing the number of lines of code needed, +so it can serve as a skeleton app for developers to +start building their application. + +## Usage + +Usage: + `./imb-burst` on x86_64 + Unexecutable on aarch64 diff --git a/examples/burst-app/main.c b/examples/burst-app/main.c new file mode 100644 index 0000000000000000000000000000000000000000..71a487af2177b1782529f20ea0a0550dc3acb6dd --- /dev/null +++ b/examples/burst-app/main.c @@ -0,0 +1,249 @@ +/********************************************************************** + Copyright(c) 2023 Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include +#include + +#include + +#define NUM_BUFS IMB_MAX_JOBS +#define BURST_SIZE 32 +#define BUF_SIZE 2048 +#define KEY_SIZE 16 +#define IV_SIZE 16 +#define AAD_SIZE 12 +#define DIGEST_SIZE 16 +#define TOTAL_NUM_JOBS 10000UL + +/* + * Fill AES-128-GCM job to be submitted with IMB_SUBMIT_BURST + */ +static void +fill_job(IMB_JOB *job, const void *src_buf, void *dst_buf, const void *iv, + const struct gcm_key_data *key, const void *aad, void *auth_tag) +{ + /* General parameters */ + job->src = src_buf; + job->dst = dst_buf; + job->chain_order = IMB_ORDER_CIPHER_HASH; + + /* Cipher parameters */ + job->cipher_mode = IMB_CIPHER_GCM; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->enc_keys = key; + job->dec_keys = key; + job->key_len_in_bytes = KEY_SIZE; + job->iv = iv; + job->iv_len_in_bytes = IV_SIZE; + job->msg_len_to_cipher_in_bytes = BUF_SIZE; + job->cipher_start_src_offset_in_bytes = 0; + + /* Authentication parameters */ + job->hash_alg = IMB_AUTH_AES_GMAC; + job->u.GCM.aad = aad; + job->u.GCM.aad_len_in_bytes = AAD_SIZE; + job->msg_len_to_hash_in_bytes = BUF_SIZE; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = auth_tag; + job->auth_tag_output_len_in_bytes = DIGEST_SIZE; +} + +static int +allocate_array(void **array, const unsigned num_elems, + const size_t elem_size) +{ + unsigned i; + + for (i = 0; i < num_elems; i++) { + array[i] = malloc(elem_size); + if (array[i] == NULL) + return -1; + } + + return 0; +} + +int main(void) +{ + unsigned i; + IMB_MGR *mb_mgr = NULL; + int exit_status = EXIT_FAILURE; + + /* Allocate buffers and authentication tags */ + void *src_bufs[NUM_BUFS]; + void *dst_bufs[NUM_BUFS]; + void *auth_tags[NUM_BUFS]; + + memset(src_bufs, 0, sizeof(src_bufs)); + memset(dst_bufs, 0, sizeof(dst_bufs)); + memset(auth_tags, 0, sizeof(auth_tags)); + + if (allocate_array(src_bufs, NUM_BUFS, BUF_SIZE) < 0) { + printf("Could not allocate memory for source buffer\n"); + goto exit; + } + if (allocate_array(dst_bufs, NUM_BUFS, BUF_SIZE) < 0) { + printf("Could not allocate memory for destination buffer\n"); + goto exit; + } + if (allocate_array(auth_tags, NUM_BUFS, DIGEST_SIZE) < 0) { + printf("Could not allocate memory for authentication tag\n"); + goto exit; + } + + /* IMB API: Allocate MB_MGR */ + mb_mgr = alloc_mb_mgr(0); + + if (mb_mgr == NULL) { + printf("Could not allocate memory for IMB_MGR\n"); + goto exit; + } + + /* IMB API: Initialize MB_MGR, detecting best implementation to use */ + init_mb_mgr_auto(mb_mgr, NULL); + + /* Prepare GCM keys (common for all buffers) */ + uint8_t key[KEY_SIZE]; + struct gcm_key_data gdata_key; + + /* IMB API: Expand AES keys and precompute GHASH keys for AES-GCM */ + IMB_AES128_GCM_PRE(mb_mgr, key, &gdata_key); + + /* Allocate memory for IV and AAD */ + uint8_t iv[BURST_SIZE][IV_SIZE]; + uint8_t aad[BURST_SIZE][AAD_SIZE]; + + /* Prepare IMB_JOB's (one job per buffer) */ + IMB_JOB *jobs[BURST_SIZE]; + unsigned completed_jobs; + unsigned total_jobs_rx = 0; + unsigned n_jobs_left = TOTAL_NUM_JOBS; + + printf("Encrypting %lu buffers with AES-GCM\n", TOTAL_NUM_JOBS); + while (n_jobs_left != 0) { + const unsigned burst_size = (n_jobs_left < BURST_SIZE) ? + n_jobs_left : BURST_SIZE; + + /* IMB API: Get next burst of IMB_JOB's */ + const unsigned n_jobs = IMB_GET_NEXT_BURST(mb_mgr, burst_size, jobs); + + /* If no jobs available, the manager needs to be flushed, to get some jobs out */ + if (n_jobs == 0) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, BURST_SIZE, jobs); + + total_jobs_rx += completed_jobs; + n_jobs_left -= completed_jobs; +#ifdef DEBUG + for (i = 0; i < completed_jobs; i++) { + if (jobs[i]->status != IMB_STATUS_COMPLETED) { + printf("Some jobs were not successful\n"); + goto exit; + } + } +#endif + continue; + } + + /* Prepare jobs */ + for (i = 0; i < n_jobs; i++) { + /* Index for next buffer in src_bufs/dst_bufs arrays */ + const unsigned buf_idx = (TOTAL_NUM_JOBS - n_jobs_left + i) % NUM_BUFS; + + fill_job(jobs[i], src_bufs[buf_idx], dst_bufs[buf_idx], + iv[i], &gdata_key, aad[i], auth_tags[buf_idx]); +#if IMB_VERSION(1, 3, 0) < IMB_VERSION_NUM + imb_set_session(mb_mgr, jobs[i]); +#endif + } + +#ifdef DEBUG + /* IMB API: Submit jobs (internally checks if job fields are correct) */ + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); + + /* IMB API: Get error number set (0 = all correct) */ + const int err = imb_get_errno(mb_mgr); + + /* IMB API: Get string for the error */ + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto exit; + } + + for (i = 0; i < completed_jobs; i++) { + if (jobs[i]->status != IMB_STATUS_COMPLETED) { + printf("Some jobs were not successful\n"); + goto exit; + } + } +#else + /* IMB API: Submit jobs (does not check the job fields, so it is faster API) */ + completed_jobs = IMB_SUBMIT_BURST_NOCHECK(mb_mgr, n_jobs, jobs); + +#endif + n_jobs_left -= n_jobs; + total_jobs_rx += completed_jobs; + } + + if (total_jobs_rx != TOTAL_NUM_JOBS) { + /* IMB API: Flush jobs (processes jobs not completed previously with + * IMB_SUBMIT_BURST/IMB_SUBMIT_BURST_NOCHECK) */ + completed_jobs = IMB_FLUSH_BURST(mb_mgr, BURST_SIZE, jobs); + + total_jobs_rx += completed_jobs; +#ifdef DEBUG + for (i = 0; i < completed_jobs; i++) { + if (jobs[i]->status != IMB_STATUS_COMPLETED) { + printf("Some jobs were not successful\n"); + goto exit; + } + } + +#endif + } + + if (total_jobs_rx != TOTAL_NUM_JOBS) { + printf("Not all jobs could be completed (expected %lu, got %u)\n", + TOTAL_NUM_JOBS, total_jobs_rx); + goto exit; + } + + exit_status = EXIT_SUCCESS; + + printf("All buffers were successfully encrypted\n"); +exit: + for (i = 0; i < NUM_BUFS; i++) { + free(src_bufs[i]); + free(dst_bufs[i]); + free(auth_tags[i]); + } + free_mb_mgr(mb_mgr); + + return exit_status; +} diff --git a/test/wycheproof/win_x64.mak b/examples/burst-app/win_x64.mak similarity index 73% rename from test/wycheproof/win_x64.mak rename to examples/burst-app/win_x64.mak index 12d3caacbecf389aeb1353846138d5949a37b22e..6d7c93cffea2c6923ce63ff2b1b34f0b5abf9e6b 100644 --- a/test/wycheproof/win_x64.mak +++ b/examples/burst-app/win_x64.mak @@ -1,98 +1,79 @@ -# -# Copyright (c) 2022, Intel Corporation -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Intel Corporation nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -APP = wycheproof -INSTNAME = intel-ipsec-mb - -!if !defined(PREFIX) -PREFIX = C:\Program Files -!endif - -!if exist("$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib") -IPSECLIB = "$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib" -INCDIR = -I"$(PREFIX)\$(INSTNAME)" -!else -!if !defined(LIB_DIR) -LIB_DIR = ..\..\lib -!endif -IPSECLIB = "$(LIB_DIR)\libIPSec_MB.lib" -INCDIR = -I$(LIB_DIR) -I.\ -!endif - -!if !defined(DEBUG_OPT) -DEBUG_OPT = /Od -!endif - -!ifdef DEBUG -DCFLAGS = $(DEBUG_OPT) /DDEBUG /Z7 -DLFLAGS = /debug -!else -DCFLAGS = /O2 /Oi -DLFLAGS = -!endif - -# compiler -CC = cl - -# _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure snprintf() being used -CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) - -#linker -LNK = link -LFLAGS = /out:$(APP).exe $(DLFLAGS) - -# dependency -!ifndef DEPTOOL -DEPTOOL = ..\..\mkdep.bat -!endif -DEPFLAGS = $(INCDIR) - -OBJS = aes_gcm_test.json.obj aes_ccm_test.json.obj \ - chacha20_poly1305_test.json.obj \ - aes_cmac_test.json.obj gmac_test.json.obj \ - hmac_sha1_test.json.obj hmac_sha224_test.json.obj \ - hmac_sha256_test.json.obj hmac_sha384_test.json.obj \ - hmac_sha512_test.json.obj wycheproof.obj - -all: $(APP).exe - -$(APP).exe: $(OBJS) $(IPSECLIB) - $(LNK) $(LFLAGS) $(OBJS) $(IPSECLIB) - -tests.dep: $(OBJS) - @type *.obj.dep > $@ 2> nul - -.c.obj: - $(CC) /c $(CFLAGS) $< - $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep - -clean: - del /q $(OBJS) tests.dep *.obj.dep $(APP).exe - -!if exist(tests.dep) -!include tests.dep -!endif +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP = imb-burst + +INSTNAME = ipsec-mb + +!if !defined(PREFIX) +PREFIX = C:\Program Files +!endif + +!if exist("$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib") +IPSECLIB = "$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib" +INCDIR = -I"$(PREFIX)\$(INSTNAME)" +!else +!if !defined(LIB_DIR) +LIB_DIR = ..\lib +!endif +IPSECLIB = "$(LIB_DIR)\libIPSec_MB.lib" +INCDIR = -I$(LIB_DIR) -I.\ +!endif + +!if !defined(DEBUG_OPT) +DEBUG_OPT = /Od +!endif + +!ifdef DEBUG +DCFLAGS = $(DEBUG_OPT) /DDEBUG /Z7 +DLFLAGS = /debug +!else +DCFLAGS = /O2 /Oi +DLFLAGS = +!endif + +CC = cl + +# _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure strtok() being used +CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) /std:c11 + +LNK = link +LFLAGS = /out:$(APP).exe $(DLFLAGS) + +OBJECTS = main.obj + +all: $(APP).exe + +$(APP).exe: $(OBJECTS) $(IPSECLIB) + $(LNK) $(LFLAGS) $(OBJECTS) $(IPSECLIB) + +.c.obj: + $(CC) /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +clean: + del /q $(OBJECTS) $(APP).exe $(APP).pdb $(APP).ilk diff --git a/examples/cmake/unix.cmake b/examples/cmake/unix.cmake new file mode 100644 index 0000000000000000000000000000000000000000..e60c857243c85eb51162306ec571d8c62e8c0362 --- /dev/null +++ b/examples/cmake/unix.cmake @@ -0,0 +1,55 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# Test application CMake Unix config +# ############################################################################## + +set(LINUX 1) +set(IPSEC_MB_LIB IPSec_MB) + +# set NASM flags +set(CMAKE_ASM_NASM_FLAGS "-felf64 -Xgnu -gdwarf -DLINUX -D__linux__") + +# set compiler definitions +set(APP_DEFINES LINUX _GNU_SOURCE) + +# set C compiler flags +set(CMAKE_C_FLAGS + "-W -Wall -Wextra -Wmissing-declarations \ +-Wpointer-arith -Wcast-qual -Wundef -Wwrite-strings -Wformat \ +-Wformat-security -Wunreachable-code -Wmissing-noreturn -Wsign-compare \ +-Wno-endif-labels -Wstrict-prototypes -Wmissing-prototypes \ +-Wold-style-definition -fno-delete-null-pointer-checks -fwrapv -std=c99") +set(CMAKE_C_FLAGS_DEBUG "-O0 -DDEBUG -g") +set(CMAKE_C_FLAGS_RELEASE "-O3") +set(CMAKE_EXE_LINKER_FLAGS "-fPIE -z noexecstack -z relro -z now") +set(CMAKE_EXE_LINKER_FLAGS_DEBUG "-g") + +# -fno-strict-overflow is not supported by clang +if(CMAKE_COMPILER_IS_GNUCC) + string(APPEND CMAKE_C_FLAGS " -fno-strict-overflow") +endif() + diff --git a/examples/cmake/windows.cmake b/examples/cmake/windows.cmake new file mode 100644 index 0000000000000000000000000000000000000000..df70a01b1929f08c831a756b8c9b4249e998ed8c --- /dev/null +++ b/examples/cmake/windows.cmake @@ -0,0 +1,41 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# Test application CMake Windows config +# ############################################################################## + +set(WINDOWS 1) +set(IPSEC_MB_LIB libIPSec_MB) + +# set NASM flags +set(CMAKE_ASM_NASM_FLAGS "-Werror -fwin64 -Xvc -DWIN_ABI") + +# set C compiler flags +set(CMAKE_C_FLAGS "/nologo /D_CRT_SECURE_NO_WARNINGS /Y- /W3 /WX- /Gm- /fp:precise /EHsc /std:c11 ${EXTRA_CFLAGS}") +set(CMAKE_C_FLAGS_DEBUG "/Od /DDEBUG /Z7") +set(CMAKE_C_FLAGS_RELEASE "/O2 /Oi") +set(CMAKE_EXE_LINKER_FLAGS_DEBUG "/debug") + diff --git a/examples/win_x64.mak b/examples/win_x64.mak new file mode 100644 index 0000000000000000000000000000000000000000..3bb29c864a389b1a9176e4067db615052343d6b2 --- /dev/null +++ b/examples/win_x64.mak @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +all: + cd burst-app & $(MAKE) /f win_x64.mak + +clean: + cd burst-app & $(MAKE) /f win_x64.mak clean diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..8954e09985adcab52d3486968291e067b45a8b6f --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,250 @@ +# Copyright (c) 2022-2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# Build IPSec_MB library +# ############################################################################## + +# ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +####################################### +# set library directories +####################################### +set(DIR_CURRENT ${CMAKE_CURRENT_SOURCE_DIR}/) +set(DIR_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/include) +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(DIR_AARCH64 ${CMAKE_CURRENT_SOURCE_DIR}/aarch64) + set(DIR_X86_64 ${CMAKE_CURRENT_SOURCE_DIR}/x86_64) + set(DIR_NO_AESNI ${CMAKE_CURRENT_SOURCE_DIR}/no-aesni) +else() + set(DIR_AVX_T1 ${CMAKE_CURRENT_SOURCE_DIR}/avx_t1) + set(DIR_AVX_T2 ${CMAKE_CURRENT_SOURCE_DIR}/avx_t2) + set(DIR_AVX2_T1 ${CMAKE_CURRENT_SOURCE_DIR}/avx2_t1) + set(DIR_AVX2_T2 ${CMAKE_CURRENT_SOURCE_DIR}/avx2_t2) + set(DIR_AVX512_T1 ${CMAKE_CURRENT_SOURCE_DIR}/avx512_t1) + set(DIR_AVX512_T2 ${CMAKE_CURRENT_SOURCE_DIR}/avx512_t2) + set(DIR_SSE_T1 ${CMAKE_CURRENT_SOURCE_DIR}/sse_t1) + set(DIR_SSE_T2 ${CMAKE_CURRENT_SOURCE_DIR}/sse_t2) + set(DIR_SSE_T3 ${CMAKE_CURRENT_SOURCE_DIR}/sse_t3) + set(DIR_X86_64 ${CMAKE_CURRENT_SOURCE_DIR}/x86_64) + set(DIR_NO_AESNI ${CMAKE_CURRENT_SOURCE_DIR}/no-aesni) + if(AVX_IFMA) + set(DIR_AVX2_T3 ${CMAKE_CURRENT_SOURCE_DIR}/avx2_t3) + endif() +endif() + +######################################## +# create list of all source directories +######################################## +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(DIR_SOURCES + ${DIR_AARCH64} + ${DIR_X86_64} + ${DIR_NO_AESNI} + ) +else() + set(DIR_SOURCES + ${DIR_AVX_T1} + ${DIR_AVX_T2} + ${DIR_AVX2_T1} + ${DIR_AVX2_T2} + ${DIR_AVX512_T1} + ${DIR_AVX512_T2} + ${DIR_SSE_T1} + ${DIR_SSE_T2} + ${DIR_SSE_T3} + ${DIR_X86_64} + ) + if(AESNI_EMU) + list(APPEND DIR_SOURCES ${DIR_NO_AESNI}) + endif() + + if(AVX_IFMA) + list(APPEND DIR_SOURCES ${DIR_AVX2_T3}) + endif() +endif() + +######################################## +# find C source files in all directories +######################################## +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + file(GLOB SRC_FILES_AARCH64 "${DIR_AARCH64}/*.c") + set(SRC_FILES_X86_64 "${DIR_X86_64}/version.c" + "${DIR_X86_64}/alloc.c" + "${DIR_X86_64}/error.c" + "${DIR_X86_64}/cipher_suite_id.c" + "${DIR_X86_64}/zuc_iv.c" + "${DIR_X86_64}/snow3g_iv.c" + "${DIR_X86_64}/snow3g_tables.c" + "${DIR_X86_64}/ooo_mgr_reset.c") + set(SRC_FILES_NO_AESNI "${DIR_NO_AESNI}/aesni_emu.c") +else() + file(GLOB SRC_FILES_AVX_T1 "${DIR_AVX_T1}/*.c") + file(GLOB SRC_FILES_AVX_T2 "${DIR_AVX_T2}/*.c") + file(GLOB SRC_FILES_AVX2_T1 "${DIR_AVX2_T1}/*.c") + file(GLOB SRC_FILES_AVX2_T2 "${DIR_AVX2_T2}/*.c") + file(GLOB SRC_FILES_AVX2_T3 "${DIR_AVX2_T3}/*.c") + file(GLOB SRC_FILES_AVX512_T1 "${DIR_AVX512_T1}/*.c") + file(GLOB SRC_FILES_AVX512_T2 "${DIR_AVX512_T2}/*.c") + file(GLOB SRC_FILES_SSE_T1 "${DIR_SSE_T1}/*.c") + file(GLOB SRC_FILES_SSE_T2 "${DIR_SSE_T2}/*.c") + file(GLOB SRC_FILES_SSE_T3 "${DIR_SSE_T3}/*.c") + file(GLOB SRC_FILES_X86_64 "${DIR_X86_64}/*.c") + file(GLOB SRC_FILES_NO_AESNI "${DIR_NO_AESNI}/*.c") +endif() + +# create list of all C source files +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(SRC_FILES_C + ${SRC_FILES_AARCH64} + ${SRC_FILES_X86_64} + ${SRC_FILES_NO_AESNI} + ) +else() + set(SRC_FILES_C + ${SRC_FILES_AVX_T1} + ${SRC_FILES_AVX_T2} + ${SRC_FILES_AVX2_T1} + ${SRC_FILES_AVX2_T2} + ${SRC_FILES_AVX512_T1} + ${SRC_FILES_AVX512_T2} + ${SRC_FILES_SSE_T1} + ${SRC_FILES_SSE_T2} + ${SRC_FILES_SSE_T3} + ${SRC_FILES_X86_64} + ) + if(AESNI_EMU) + list(APPEND SRC_FILES_C ${SRC_FILES_NO_AESNI}) + endif() + if(AVX_IFMA) + list(APPEND SRC_FILES_C ${SRC_FILES_AVX2_T3}) + endif() +endif() + +list(SORT SRC_FILES_C) + +######################################## +# generate list of assembly source files +######################################## +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + file(GLOB_RECURSE TMP "${DIR_AARCH64}/*.S") + list(APPEND SRC_FILES_ASM ${TMP}) +else() + foreach(DIR ${DIR_SOURCES}) + file(GLOB_RECURSE TMP "${DIR}/*.asm") + list(APPEND SRC_FILES_ASM ${TMP}) + endforeach() +endif() + +list(SORT SRC_FILES_ASM) + +######################################## +# set C compiler options +######################################## +set(LIB_DEFINES) + +# enable SAFE OPTIONS +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + if(SAFE_DATA) + list(APPEND LIB_DEFINES SAFE_DATA) + string(APPEND CMAKE_ASM_FLAGS " -DSAFE_DATA") + endif() + if(SAFE_PARAM) + list(APPEND LIB_DEFINES SAFE_PARAM) + string(APPEND CMAKE_ASM_FLAGS " -DSAFE_PARAM") + endif() + if(SAFE_LOOKUP) + list(APPEND LIB_DEFINES SAFE_LOOKUP) + string(APPEND CMAKE_ASM_FLAGS " -DSAFE_LOOKUP") + endif() + if(AESNI_EMU) + list(APPEND LIB_DEFINES AESNI_EMU) + string(APPEND CMAKE_ASM_FLAGS " -DAESNI_EMU") + endif() +else() + if(SAFE_DATA) + list(APPEND LIB_DEFINES SAFE_DATA) + string(APPEND CMAKE_ASM_NASM_FLAGS " -DSAFE_DATA") + endif() + if(SAFE_PARAM) + list(APPEND LIB_DEFINES SAFE_PARAM) + string(APPEND CMAKE_ASM_NASM_FLAGS " -DSAFE_PARAM") + endif() + if(SAFE_LOOKUP) + list(APPEND LIB_DEFINES SAFE_LOOKUP) + string(APPEND CMAKE_ASM_NASM_FLAGS " -DSAFE_LOOKUP") + endif() + if(AESNI_EMU) + list(APPEND LIB_DEFINES AESNI_EMU) + string(APPEND CMAKE_ASM_NASM_FLAGS " -DAESNI_EMU") + endif() +endif() + +# enable AVX IFMA support +if(AVX_IFMA) + list(APPEND LIB_DEFINES AVX_IFMA) +endif() + +######################################## +# setup NASM +######################################## +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + enable_language(ASM) + if(NOT CMAKE_ASM_COMPILER_LOADED) + message(FATAL_ERROR "Can't find assembler") + endif() +else() + enable_language(ASM_NASM) + if(NOT CMAKE_ASM_NASM_COMPILER_LOADED) + message(FATAL_ERROR "Can't find assembler") + endif() +endif() +set(CAN_USE_ASSEMBLER 1) + +######################################## +# add OS specific options +######################################## +if(WINDOWS) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/windows.cmake) +else() + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/unix.cmake) +endif() + +# add user defined cflags +string(APPEND CMAKE_C_FLAGS " ${EXTRA_CFLAGS}") + +# add library definitions to all C source files +foreach(FILE ${SRC_FILES_C}) + set_source_files_properties(${FILE} PROPERTIES + COMPILE_DEFINITIONS "${LIB_DEFINES}") +endforeach() + +# add include directories +target_include_directories(${LIB} PRIVATE + ${DIR_CURRENT} ${DIR_INCLUDE} ${DIR_NO_AESNI}) + diff --git a/lib/Makefile b/lib/Makefile index 6da9a17fcf331ca4941a1ed65464750fe8d2f606..ab8cc65ba56379fa1e860b4647c0d4b435464579 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -24,6 +24,9 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # + +ARCH = $(shell uname -m) + LIB = libIPSec_MB SHARED ?= y IMB_HDR = ipsec-mb.h @@ -34,8 +37,6 @@ ifeq ($(IMB_VERSION),) $(error "Failed to detect library version!") endif -ARCH = $(shell uname -m) - VERSION = $(shell echo $(IMB_VERSION) | cut -d. -f1-3) SO_VERSION = $(shell echo $(VERSION) | cut -d. -f1) @@ -51,48 +52,72 @@ ifeq ($(ARCH),x86_64) USE_YASM ?= n YASM ?= yasm NASM ?= nasm +endif + +CC ?= gcc + +OBJ_DIR ?= obj +LIB_DIR ?= . + +# skip version detection with clean or style target +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),style) + +# MINGW should be non-zero value if detected +MINGW ?= $(shell $(CC) -dM -E - < /dev/null | grep -i mingw | wc -l | sed 's/^ *//') + +ifeq ($(MINGW),0) +MULT = \* +else +MULT = \\* +endif +ifeq ($(ARCH),x86_64) # Detect NASM version (minimum version required: 2.14) NASM_VERSION = $(shell $(NASM) -v | cut -d " " -f 3) NASM_MAJOR_REQ = 2 NASM_MINOR_REQ = 14 +NASM_MIN_REQ = $(shell expr $(NASM_MAJOR_REQ) $(MULT) 100 + $(NASM_MINOR_REQ) ) ifeq ($(NASM_VERSION),) $(error "NASM is not installed! Minimum required version: $(NASM_MAJOR_REQ).$(NASM_MINOR_REQ)") else NASM_MAJOR_VER = $(shell echo $(NASM_VERSION) | cut -d "." -f 1) NASM_MINOR_VER = $(shell echo $(NASM_VERSION) | cut -d "." -f 2 | cut -c 1-2) -NASM_GE_MAJOR = $(shell [ $(NASM_MAJOR_VER) -ge $(NASM_MAJOR_REQ) ] && echo true) -NASM_GE_MINOR = $(shell [ $(NASM_MINOR_VER) -ge $(NASM_MINOR_REQ) ] && echo true) -ifneq ($(NASM_GE_MAJOR),true) -$(warning "NASM version found: $(NASM_VERSION)") -$(error "Minimum required: $(NASM_MAJOR_REQ).$(NASM_MINOR_REQ)") -endif # NASM_GE_MAJOR -ifneq ($(NASM_GE_MINOR),true) -$(warning "NASM version found: $(NASM_VERSION)") -$(error "Minimum required: $(NASM_MAJOR_REQ).$(NASM_MINOR_REQ)") -endif # NASM_GE_MINOR +NASM_VER = $(shell expr $(NASM_MAJOR_VER) $(MULT) 100 + $(NASM_MINOR_VER) ) + +NASM_GE_REQ = $(shell [ $(NASM_VER) -ge $(NASM_MIN_REQ) ] && echo true) + +$(info NASM '$(NASM)' version: $(NASM_VERSION)) + +ifneq ($(NASM_GE_REQ),true) +$(error "Minimum required NASM version: $(NASM_MAJOR_REQ).$(NASM_MINOR_REQ)") +endif # NASM_GE_REQ endif # NASM_VERSION +# Minimum version of NASM with AVX_IFMA support: 2.16 +AVX_IFMA := y +NASM_MAJOR_AVX_IFMA = 2 +NASM_MINOR_AVX_IFMA = 16 +NASM_MIN_IFMA_REQ = $(shell expr $(NASM_MAJOR_AVX_IFMA) $(MULT) 100 + $(NASM_MINOR_AVX_IFMA) ) + +NASM_IFMA_GE_REQ = $(shell [ $(NASM_VER) -ge $(NASM_MIN_IFMA_REQ) ] && echo true) +ifneq ($(NASM_IFMA_GE_REQ),true) +$(warning Minimum required NASM version for AVX-IFMA: $(NASM_MAJOR_AVX_IFMA).$(NASM_MINOR_AVX_IFMA). AVX-IFMA code not compiled - update NASM.) +AVX_IFMA := n +endif # NASM_AVX_IFMA_GET_REQ endif # x86_64 -OBJ_DIR ?= obj -LIB_DIR ?= . - INCLUDE_DIRS := include . no-aesni INCLUDES := $(foreach i,$(INCLUDE_DIRS),-I $i) -CC ?= gcc ifneq ($(filter %gcc,$(CC)),) STRIP ?= $(CC:gcc=strip) else STRIP ?= strip endif -# MINGW should be non-zero value if detected -MINGW ?= $(shell $(CC) -dM -E - < /dev/null | grep -i mingw | wc -l | sed 's/^ *//') - # if "-z ibt" is supported then assume "-z shstk, -z cet-report=error" are also supported # "-fcf-protection" needs to be checked separately ifeq ($(ARCH),x86_64) @@ -100,15 +125,24 @@ ifeq ($(MINGW),0) CC_HAS_CET = $(and $(shell $(CC) --target-help 2> /dev/null | grep -m1 -e "-z ibt" | wc -l), \ $(shell $(CC) --help=common 2> /dev/null | grep -m1 -e "-fcf-protection" | wc -l)) CET_LDFLAGS=-r -z ibt -z shstk -endif # MINGW -endif # x86_64 -CFLAGS := -fPIC -DNO_COMPAT_IMB_API_053 $(EXTRA_CFLAGS) $(INCLUDES) \ +endif +CFLAGS := -fPIC $(EXTRA_CFLAGS) $(INCLUDES) \ -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ -Wcast-qual -Wundef -Wwrite-strings \ -Wformat -Wformat-security \ -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ - -fno-delete-null-pointer-checks -fwrapv + -fno-delete-null-pointer-checks -fwrapv -std=c99 +else # x86_64 +CFLAGS := -fPIC $(EXTRA_CFLAGS) $(INCLUDES) \ + -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ + -Wcast-qual -Wundef -Wwrite-strings \ + -Wformat -Wformat-security \ + -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ + -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ + -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function \ + -fno-delete-null-pointer-checks -fwrapv -std=c99 +endif # -fno-strict-overflow is not supported by clang ifneq ($(CC),clang) @@ -125,6 +159,10 @@ CFLAGS += -fcf-protection=full endif endif # x86_64 +ifeq ($(AVX_IFMA), y) +CFLAGS += -DAVX_IFMA +endif + ASM_INCLUDE_DIRS := . YASM_INCLUDES := $(foreach i,$(ASM_INCLUDE_DIRS),-I $i) @@ -135,7 +173,7 @@ NASM_FLAGS := -Werror -fwin64 -Xvc -gcv8 -DWIN_ABI $(NASM_INCLUDES) else YASM_FLAGS := -f x64 -f elf64 -X gnu -g dwarf2 -DLINUX -D__linux__ $(YASM_INCLUDES) NASM_FLAGS := -Werror -felf64 -Xgnu -gdwarf -DLINUX -D__linux__ $(NASM_INCLUDES) -endif # MINGW +endif DEBUG_OPT ?= -O0 ifeq ($(DEBUG),y) @@ -149,8 +187,8 @@ CFLAGS += -fstack-protector -D_FORTIFY_SOURCE=2 else OPT = -O2 LDFLAGS += -s -endif # MINGW -endif # DEBUG +endif +endif ifeq ($(SAFE_OPTIONS), n) SAFE_DATA = n @@ -197,15 +235,15 @@ OPT_NOAESNI := -msse4.2 -mno-aes -mno-pclmul endif # x86_64 ifeq ($(ARCH),aarch64) -OPT_AARCH64 := -march=armv8-a+crypto+aes -OPT_SVE := -march=armv8-a+sve+crypto+aes -OPT_NOAESNI := -march=armv8-a +OPT_AARCH64 := -march=armv8-a+crypto+aes+crc +OPT_NOAESNI := -march=armv8-a+crypto+crc endif # aarch64 # Set architectural optimizations for GCC/CC ifeq ($(CC),$(filter $(CC),gcc cc)) GCC_VERSION = $(shell $(CC) -dumpversion | cut -d. -f1) GCC_GE_V5 = $(shell [ $(GCC_VERSION) -ge 5 ] && echo true) +$(info CC '$(CC)' version: $(GCC_VERSION)) ifeq ($(GCC_GE_V5),true) ifeq ($(ARCH),aarch64) GCC_GE_V11 = $(shell [ $(GCC_VERSION) -ge 11 ] && echo true) @@ -214,18 +252,17 @@ ifneq ($(GCC_GE_V11),true) $(warning "GCC version found: $(GCC_VERSION)") $(error "Minimum required: 11") endif # GCC_GE_V11 -OPT_AARCH64 := -march=armv8-a+crypto+aes -OPT_SVE := -march=armv8-a+sve+crypto+aes -OPT_NOAESNI := -march=armv8-a +OPT_AARCH64 := -march=armv8-a+crypto+aes+crc +OPT_NOAESNI := -march=armv8-a+crypto+crc else OPT_SSE := -march=nehalem -maes -mpclmul OPT_AVX := -march=sandybridge -maes -mpclmul OPT_AVX2 := -march=haswell -maes -mpclmul OPT_AVX512 := -march=broadwell -maes -mpclmul OPT_NOAESNI := -march=nehalem -mno-pclmul -endif # AARCH64 -endif # GCC -endif # CC +endif # aarch64 +endif +endif ifeq ($(ARCH),aarch64) ifeq ($(CC),$(filter $(CC),clang)) @@ -238,26 +275,27 @@ $(warning "CLANG version found: $(CLANG_VERSION)") $(error "Minimum required: 11.0.0") endif # CLANG_GE_V11 endif # CLANG -endif # AARCH64 +endif # aarch64 # Set architectural optimizations for clang ifeq ($(CC),$(filter $(CC),clang)) CLANG_VERSION = $(shell $(CC) --version | head -n 1 | cut -d ' ' -f 3) CLANG_GE_V381 = $(shell test "$(CLANG_VERSION)" \> "3.8.0" && echo true) +$(info CLANG '$(CC)' version: $(CLANG_VERSION)) ifeq ($(CLANG_GE_V381),true) ifeq ($(ARCH),aarch64) -OPT_AARCH64 := -march=armv8-a+crypto+aes +OPT_AARCH64 := -march=armv8-a+crypto+aes+crc else OPT_SSE := -march=nehalem -maes -mpclmul OPT_AVX := -march=sandybridge -maes -mpclmul OPT_AVX2 := -march=haswell -maes -mpclmul OPT_AVX512 := -march=broadwell -maes -mpclmul -endif # AARCH64 -endif # CLANG +endif # aarch64 +endif # remove CFLAGS that clang warns about CFLAGS := $(subst -fno-delete-null-pointer-checks,,$(CFLAGS)) CFLAGS := $(subst -fno-strict-overflow,,$(CFLAGS)) -endif # CC +endif # so or static build ifeq ($(SHARED),y) @@ -266,7 +304,7 @@ LIBNAME = $(LIB).dll else LIBNAME = $(LIB).so.$(VERSION) LDFLAGS += -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now -endif # MINGW +endif LIBPERM = 0755 ifeq ($(ARCH),x86_64) ifeq ($(CC_HAS_CET),1) @@ -281,8 +319,11 @@ ifeq ($(ARCH),x86_64) ifeq ($(CC_HAS_CET),1) LDFLAGS += -fcf-protection=full endif +endif # x86_64 endif -endif # shared + +endif # style target +endif # clean target # warning messages SAFE_PARAM_MSG1="SAFE_PARAM option not set." @@ -306,7 +347,7 @@ c_lib_objs := \ mb_mgr_aarch64_sve256.o \ mb_mgr_aarch64_no_aesni.o \ mb_mgr_auto_aarch64.o \ - alloc_aarch64.o \ + alloc.o \ clear_mem_aarch64.o \ cpu_features_aarch64.o \ version.o \ @@ -317,20 +358,23 @@ c_lib_objs := \ snow3g_iv.o \ error.o \ zuc_iv.o \ - zuc_simd.o \ zuc_aarch64_no_aesni_top.o \ - zuc_simd_no_aesni.o \ zuc_aarch64_top.o \ + crc32_aarch64.o \ mb_mgr_zuc_submit_flush_aarch64.o \ mb_mgr_zuc_submit_flush_aarch64_no_aesni.o \ mb_mgr_snow3g_submit_flush_aarch64.o \ mb_mgr_snow3g_submit_flush_aarch64_no_aesni.o \ mb_mgr_snow3g_submit_flush_aarch64_sve256.o \ snow3g_aarch64_sve256.o \ - snow3g_impl_aarch64_sve256.o \ - snow3g_impl_aarch64_neon.o + cipher_suite_id.o \ + ooo_mgr_reset.o asm_generic_lib_objs := \ - lookup_16x8bit_neon.o + lookup_16x8bit_neon.o \ + snow3g_impl_aarch64_neon.o \ + snow3g_impl_aarch64_sve256.o \ + zuc_simd.o \ + zuc_simd_no_aesni.o else c_lib_objs := \ mb_mgr_avx.o \ @@ -350,9 +394,12 @@ c_lib_objs := \ aes_xcbc_expand_key.o \ md5_one_block.o \ sha_sse.o \ + sha_ni_sse.o \ sha_mb_sse.o \ sha_ni_mb_sse.o \ sha_avx.o \ + sha_avx2.o \ + sha_avx512.o \ sha_mb_avx.o \ sha_mb_avx2.o \ sha_mb_avx512.o \ @@ -377,7 +424,11 @@ c_lib_objs := \ mb_mgr_auto.o \ error.o \ ooo_mgr_reset.o \ - self_test.o + self_test.o \ + quic_aes_gcm.o \ + quic_hp_aes_ecb.o \ + hmac_ipad_opad.o \ + cipher_suite_id.o ifeq ($(AESNI_EMU), y) c_lib_objs := $(c_lib_objs) \ @@ -387,6 +438,10 @@ c_lib_objs := $(c_lib_objs) \ snow3g_sse_no_aesni.o endif +ifeq ($(AVX_IFMA), y) +c_lib_objs := $(c_lib_objs) \ + mb_mgr_avx2_t3.o +endif # # List of ASM modules (root directory/common) # @@ -405,7 +460,10 @@ asm_generic_lib_objs := \ crc32_refl_const.o \ crc32_const.o \ poly1305.o \ - chacha20_poly1305.o + chacha20_poly1305.o \ + mbcpuid.o \ + atomic.o + # # List of ASM modules (no-aesni directory) # @@ -477,6 +535,7 @@ asm_sse_lib_objs := \ aes128_cntr_by8_sse.o \ aes192_cntr_by8_sse.o \ aes256_cntr_by8_sse.o \ + aes_ecb_quic_x8_sse.o \ aes128_ecb_by4_sse.o \ aes192_ecb_by4_sse.o \ aes256_ecb_by4_sse.o \ @@ -498,6 +557,8 @@ asm_sse_lib_objs := \ sha256_one_block_sse.o \ sha384_one_block_sse.o \ sha512_one_block_sse.o \ + sha1_ni_one_block_sse.o \ + sha256_ni_one_block_sse.o \ sha512_x2_sse.o \ sha256_mult_sse.o \ sha1_ni_x2_sse.o \ @@ -587,6 +648,7 @@ asm_avx_lib_objs := \ aes128_ecb_by8_avx.o \ aes192_ecb_by8_avx.o \ aes256_ecb_by8_avx.o \ + aes_ecb_quic_x8_avx.o \ aes_cfb_avx.o \ aes128_cbc_mac_x8_avx.o \ aes256_cbc_mac_x8_avx.o \ @@ -653,6 +715,7 @@ asm_avx2_lib_objs := \ sha256_oct_avx2.o \ sha512_x4_avx2.o \ zuc_x8_avx2.o \ + zuc_x8_gfni_avx2.o \ aes128_ecb_vaes_avx2.o \ aes192_ecb_vaes_avx2.o \ aes256_ecb_vaes_avx2.o \ @@ -669,8 +732,15 @@ asm_avx2_lib_objs := \ mb_mgr_hmac_sha512_flush_avx2.o \ mb_mgr_hmac_sha512_submit_avx2.o \ mb_mgr_zuc_submit_flush_avx2.o \ + mb_mgr_zuc_submit_flush_gfni_avx2.o \ chacha20_avx2.o +# +# List of ASM modules (avx2_t3 directory) +# +asm_avx2_ifma_lib_objs := \ + poly_fma_avx2.o + # # List of ASM modules (avx512 directory) # @@ -680,6 +750,7 @@ asm_avx512_lib_objs := \ sha512_x8_avx512.o \ des_x16_avx512.o \ aes_ecb_vaes_avx512.o \ + aes_ecb_quic_vaes_avx512.o \ aes_cntr_api_by16_vaes_avx512.o \ aes_cntr_bit_api_by16_vaes_avx512.o \ aes_cntr_ccm_api_by16_vaes_avx512.o \ @@ -720,8 +791,8 @@ asm_avx512_lib_objs := \ mb_mgr_zuc_submit_flush_avx512.o \ mb_mgr_zuc_submit_flush_gfni_avx512.o \ chacha20_avx512.o \ - poly_avx512.o \ - poly_fma_avx512.o \ + poly_avx512.o \ + poly_fma_avx512.o \ ethernet_fcs_avx512.o \ crc16_x25_avx512.o \ crc32_refl_by16_vclmul_avx512.o \ @@ -753,19 +824,18 @@ asm_sse_gcm_objs := \ gcm128_sgl_api_by8_sse.o gcm192_sgl_api_by8_sse.o gcm256_sgl_api_by8_sse.o \ gcm128_gmac_api_by8_sse.o gcm192_gmac_api_by8_sse.o gcm256_gmac_api_by8_sse.o -asm_avx_gcm_objs := \ - aes128_gcm_by8_avx.o aes192_gcm_by8_avx.o aes256_gcm_by8_avx.o +asm_avx_gcm_objs := asm_avx2_gcm_objs := \ aes128_gcm_by8_avx2.o aes192_gcm_by8_avx2.o aes256_gcm_by8_avx2.o asm_avx512_gcm_objs := \ - aes128_gcm_by48_api_vaes_avx512.o aes192_gcm_by48_api_vaes_avx512.o aes256_gcm_by48_api_vaes_avx512.o \ - aes128_gcm_by48_sgl_api_vaes_avx512.o aes192_gcm_by48_sgl_api_vaes_avx512.o aes256_gcm_by48_sgl_api_vaes_avx512.o \ - aes128_gmac_by48_api_vaes_avx512.o aes192_gmac_by48_api_vaes_avx512.o aes256_gmac_by48_api_vaes_avx512.o \ + aes128_gcm_api_vaes_avx512.o aes192_gcm_api_vaes_avx512.o aes256_gcm_api_vaes_avx512.o \ + aes128_gcm_sgl_api_vaes_avx512.o aes192_gcm_sgl_api_vaes_avx512.o aes256_gcm_sgl_api_vaes_avx512.o \ + aes128_gmac_api_vaes_avx512.o aes192_gmac_api_vaes_avx512.o aes256_gmac_api_vaes_avx512.o \ aes128_gcm_by8_avx512.o aes192_gcm_by8_avx512.o aes256_gcm_by8_avx512.o -endif # AARCH64 +endif # aarch64 # # build object files lists @@ -783,8 +853,12 @@ asm_obj_files := $(asm_generic_lib_objs) \ ifeq ($(AESNI_EMU), y) asm_obj_files := $(asm_obj_files) $(asm_noaesni_lib_objs) $(asm_noaesni_gcm_objs) endif +ifeq ($(AVX_IFMA), y) +asm_obj_files := $(asm_obj_files) $(asm_avx2_ifma_lib_objs) +endif + c_obj_files := $(c_lib_objs) $(c_gcm_objs) -endif # AARCH64 +endif # aarch64 # # aggregate all objects files together and prefix with OBJDIR @@ -810,9 +884,22 @@ dep_target_files := $(dep_files:%=$(OBJ_DIR)/%) all: $(LIB_DIR)/$(LIBNAME) +STR_FILTER = "" +ifneq ($(AESNI_EMU),y) +ifneq ($(AVX_IFMA),y) +STR_FILTER = "_no_aesni\|_avx2_t3" +else +STR_FILTER = "_no_aesni" +endif +else +ifneq ($(AVX_IFMA),y) +STR_FILTER = "_avx2_t3" +endif +endif + $(LIB)_lnk.def: $(LIB).def -ifneq ($(AESNI_EMU), y) - grep -v _no_aesni $(LIB).def > $(LIB)_lnk.def +ifneq ($(STR_FILTER), "") + grep -v $(STR_FILTER) $(LIB).def > $(LIB)_lnk.def else cp -f $(LIB).def $(LIB)_lnk.def endif @@ -889,8 +976,6 @@ $(dep_target_files): | $(OBJ_DIR) # ifeq ($(ARCH),aarch64) -$(OBJ_DIR)/cpu_features_aarch64.o:aarch64/cpu_features_aarch64.c - $(CC) -MMD $(OPT_SVE) -c $(CFLAGS) $< -o $@ $(OBJ_DIR)/%.o:aarch64/%.c $(CC) -MMD $(OPT_AARCH64) -c $(CFLAGS) $< -o $@ $(OBJ_DIR)/%.o:x86_64/%.c @@ -1012,6 +1097,20 @@ ifeq ($(CC_HAS_CET),1) mv $@.tmp $@ endif +$(OBJ_DIR)/%.o:avx2_t3/%.c + $(CC) -MMD $(OPT_AVX2) -c $(CFLAGS) $< -o $@ + +$(OBJ_DIR)/%.o:avx2_t3/%.asm +ifeq ($(USE_YASM),y) + $(YASM) $(YASM_FLAGS) $< -o $@ +else + $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< +endif +ifeq ($(CC_HAS_CET),1) + $(LD) $(CET_LDFLAGS) -o $@.tmp $@ + mv $@.tmp $@ +endif + $(OBJ_DIR)/%.o:avx512_t1/%.c $(CC) -MMD $(OPT_AVX512) -c $(CFLAGS) $< -o $@ @@ -1052,8 +1151,8 @@ endif ifeq ($(CC_HAS_CET),1) $(LD) $(CET_LDFLAGS) -o $@.tmp $@ mv $@.tmp $@ -endif # CC_HAS_CET -endif # AARCH64 +endif +endif # aarch64 $(OBJ_DIR): mkdir $(OBJ_DIR) @@ -1069,10 +1168,9 @@ TAGS: .PHONY: clean clean: - rm -Rf $(target_obj_files) - rm -Rf $(dep_target_files) - rm -f $(LIB_DIR)/$(LIB)_lnk.def - rm -f $(LIB_DIR)/$(LIB).a $(LIB_DIR)/$(LIB).so* $(LIB_DIR)/$(LIB).dll + -rm -Rf $(OBJ_DIR) + -rm -f $(LIB_DIR)/$(LIB)_lnk.def + -rm -f $(LIB_DIR)/$(LIB).a $(LIB_DIR)/$(LIB).so* $(LIB_DIR)/$(LIB).dll .PHONY: doxy doxy: @@ -1112,6 +1210,7 @@ help: @echo "SAFE_OPTIONS=n" @echo " - Disable all safe options (enabled by default)" + ifneq ($(ARCH), aarch64) CHECKPATCH ?= checkpatch.pl # checkpatch ignore settings: @@ -1125,7 +1224,7 @@ CHECKPATCH ?= checkpatch.pl # # NOTE: these flags cannot be broken into multiple lines due to # spaces injected by make -CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,NEW_TYPEDEFS,AVOID_EXTERNS,COMPLEX_MACRO,PREFER_ALIGNED,USE_FUNC,CONSTANT_COMPARISON,SPACING,GCC_BINARY_CONSTANT +CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,NEW_TYPEDEFS,AVOID_EXTERNS,COMPLEX_MACRO,PREFER_ALIGNED,USE_FUNC,CONSTANT_COMPARISON,SPACING,GCC_BINARY_CONSTANT,VOLATILE %.c_style_check : %.c $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< @@ -1140,13 +1239,29 @@ CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --ignore CODE_INDEN $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< SOURCES_DIRS := . sse avx avx2 avx512 include no-aesni -SOURCES := $(foreach dir,$(SOURCES_DIRS),$(wildcard $(dir)/*.[ch]) $(wildcard $(dir)/*.asm) $(wildcard $(dir)/*.inc)) +SOURCES := $(foreach dir,$(SOURCES_DIRS),$(wildcard $(dir)/**/*.[ch]) $(wildcard $(dir)/**/*.asm) $(wildcard $(dir)/**/*.inc)) SOURCES_STYLE := $(foreach infile,$(SOURCES),$(infile)_style_check) -endif # AARCH64 +endif # aarch64 .PHONY: style style: $(SOURCES_STYLE) +# cppcheck analysis check +CPPCHECK ?= cppcheck +CPPCHECK_OPTS ?= -I./include -I./ +CPPCHECK_FLAGS = -j $(shell getconf _NPROCESSORS_ONLN) +CPPCHECK_FLAGS1 = --cppcheck-build-dir=.cppcheck $(CPPCHECK_FLAGS) +CPPCHECK_FLAGS2 = --cppcheck-build-dir=.bughunt $(CPPCHECK_FLAGS) +.PHONY: cppcheck +cppcheck: + mkdir -p .cppcheck + $(CPPCHECK) --force --enable=all $(CPPCHECK_FLAGS1) $(CPPCHECK_OPTS) ./ + +.PHONY: bughunt +bughunt: + mkdir -p .bughunt + $(CPPCHECK) --bug-hunting --inconclusive $(CPPCHECK_FLAGS2) $(CPPCHECK_OPTS) ./ + # if target not clean or rinse then make dependencies ifneq ($(MAKECMDGOALS),clean) ifneq ($(MAKECMDGOALS),style) diff --git a/lib/aarch64/aesni_emu_aarch64.S b/lib/aarch64/aesni_emu_aarch64.inc similarity index 100% rename from lib/aarch64/aesni_emu_aarch64.S rename to lib/aarch64/aesni_emu_aarch64.inc diff --git a/lib/aarch64/alloc_aarch64.c b/lib/aarch64/alloc_aarch64.c deleted file mode 100644 index eb303b6c46921e1008de5d62e7dce31be7f903ab..0000000000000000000000000000000000000000 --- a/lib/aarch64/alloc_aarch64.c +++ /dev/null @@ -1,250 +0,0 @@ -/********************************************************************** - Copyright(c) 2021-2022 Arm Corporation All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Arm Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -**********************************************************************/ -#include - -#include /* posix_memalign() and free() */ - -#include -#include /* offsetof() */ -#include "ipsec-mb.h" -#include "ipsec_ooo_mgr.h" -#include "cpu_feature.h" -#include "error.h" - -#define IMB_OOO_ROAD_BLOCK 0xDEADCAFEDEADCAFEULL - -#define ALIGNMENT 64 -#define ALIGN(x, y) ((x + (y - 1)) & (~(y - 1))) - -#define OOO_INFO(imb_mgr_ooo_ptr_name__, ooo_mgr_type__) \ - { offsetof(IMB_MGR, imb_mgr_ooo_ptr_name__), \ - ALIGN(sizeof(ooo_mgr_type__), ALIGNMENT), \ - offsetof(ooo_mgr_type__, road_block) } - -const struct { - size_t ooo_ptr_offset; - size_t ooo_aligned_size; - size_t road_block_offset; -} ooo_mgr_table[] = { - OOO_INFO(zuc_eea3_ooo, MB_MGR_ZUC_OOO), - OOO_INFO(zuc_eia3_ooo, MB_MGR_ZUC_OOO), - OOO_INFO(zuc256_eea3_ooo, MB_MGR_ZUC_OOO), - OOO_INFO(zuc256_eia3_ooo, MB_MGR_ZUC_OOO), - OOO_INFO(snow3g_uea2_ooo, MB_MGR_SNOW3G_OOO), - OOO_INFO(snow3g_uia2_ooo, MB_MGR_SNOW3G_OOO), -}; - -/** - * @brief Calculates necessary memory size for IMB_MGR. - * - * @return Size for IMB_MGR (aligned to 64 bytes) - */ -size_t imb_get_mb_mgr_size(void) -{ - size_t ooo_total_size = 0; - unsigned i; - - for (i = 0; i < IMB_DIM(ooo_mgr_table); i++) - ooo_total_size += ooo_mgr_table[i].ooo_aligned_size; - /* - * Add 64 bytes into the maximum size calculation to - * make sure there is enough room to align the OOO managers. - */ - return (sizeof(IMB_MGR) + ooo_total_size + ALIGNMENT); -} - -static uint8_t *get_ooo_ptr(IMB_MGR *mgr, const size_t offset) -{ - uint8_t *mgr_offset = &((uint8_t *) mgr)[offset]; - uint8_t **ptr = (uint8_t **) mgr_offset; - - return *ptr; -} - -static void set_ooo_ptr(IMB_MGR *mgr, const size_t offset, uint8_t *new_ptr) -{ - uint8_t *mgr_offset = &((uint8_t *) mgr)[offset]; - uint8_t **ptr = (uint8_t **) mgr_offset; - - *ptr = new_ptr; -} - -static void set_road_block(uint8_t *ooo_ptr, const size_t offset) -{ - uint64_t *p_road_block = (uint64_t *) &ooo_ptr[offset]; - - *p_road_block = IMB_OOO_ROAD_BLOCK; -} - -/* - * Set last 8 bytes of OOO mgrs to predefined pattern - * - * This is to assist in searching for sensitive data remaining - * in the heap after algorithmic code completes - */ -static void set_ooo_mgr_road_block(IMB_MGR *mgr) -{ - unsigned n; - - for (n = 0; n < IMB_DIM(ooo_mgr_table); n++) - set_road_block(get_ooo_ptr(mgr, - ooo_mgr_table[n].ooo_ptr_offset), - ooo_mgr_table[n].road_block_offset); -} - - -/** - * @brief Initializes IMB_MGR pointers to out-of-order managers with - * use of externally allocated memory. - * - * imb_get_mb_mgr_size() should be called to know how much memory - * should be allocated externally. - * - * init_mb_mgr_XXX() must be called after this function call, - * whereas XXX is the desired architecture (including "auto"), - * only if reset_mgr is set to 0. - * - * @param mem_ptr a pointer to allocated memory - * @param flags multi-buffer manager flags - * IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions. - * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. - * - * @param reset_mgr if 0, IMB_MGR structure is not cleared, else it is. - * - * @return Pointer to IMB_MGR structure - */ -IMB_MGR *imb_set_pointers_mb_mgr(void *mem_ptr, const uint64_t flags, - const unsigned reset_mgr) -{ - if (mem_ptr == NULL) { - imb_set_errno(mem_ptr, ENOMEM); - return NULL; - } - - IMB_MGR *ptr = (IMB_MGR *) mem_ptr; - uint8_t *ptr8 = (uint8_t *) ptr; - uint8_t *free_mem = &ptr8[ALIGN(sizeof(IMB_MGR), ALIGNMENT)]; - const size_t mem_size = imb_get_mb_mgr_size(); - unsigned i; - - if (reset_mgr) { - /* Zero out MB_MGR memory */ - memset(mem_ptr, 0, mem_size); - } else { - IMB_ARCH used_arch = (IMB_ARCH) ptr->used_arch; - - /* Reset function pointers from previously used architecture */ - switch (used_arch) { - case IMB_ARCH_NOAESNI: - init_mb_mgr_aarch64_no_aesni_internal(ptr, 0); - break; - case IMB_ARCH_AARCH64: - init_mb_mgr_aarch64_internal(ptr, 0); - break; - default: - break; - } - } - - imb_set_errno(ptr, 0); - ptr->flags = flags; /* save the flags for future use in init */ - ptr->features = cpu_feature_adjust(flags, cpu_feature_detect()); - - /* Set OOO pointers */ - for (i = 0; i < IMB_DIM(ooo_mgr_table); i++) { - set_ooo_ptr(ptr, ooo_mgr_table[i].ooo_ptr_offset, free_mem); - free_mem = &free_mem[ooo_mgr_table[i].ooo_aligned_size]; - IMB_ASSERT((uintptr_t)(free_mem - ptr8) <= mem_size); - } - set_ooo_mgr_road_block(ptr); - - return ptr; -} - - -static void * -alloc_aligned_mem(const size_t size) -{ - void *ptr; - - const size_t alignment = 64; - if (posix_memalign((void **)&ptr, alignment, size)) - return NULL; - - IMB_ASSERT(ptr != NULL); - - memset(ptr, 0, size); - - return ptr; -} - -static void -free_mem(void *ptr) -{ - free(ptr); -} - -/** - * @brief Allocates memory for multi-buffer manager instance - * - * For binary compatibility between library versions - * it is recommended to use this API. - * - * @return Pointer to allocated memory for MB_MGR structure - * @retval NULL on allocation error - */ -IMB_MGR *alloc_mb_mgr(uint64_t flags) -{ - IMB_MGR *ptr = NULL; - - ptr = alloc_aligned_mem(imb_get_mb_mgr_size()); - IMB_ASSERT(ptr != NULL); - if (ptr != NULL) { - imb_set_pointers_mb_mgr(ptr, flags, 1); - } else { - imb_set_errno(ptr, ENOMEM); - return NULL; - } - - return ptr; -} - -/** - * @brief Frees memory allocated previously by alloc_mb_mgr() - * - * @param ptr a pointer to allocated MB_MGR structure - * - */ -void free_mb_mgr(IMB_MGR *ptr) -{ - IMB_ASSERT(ptr != NULL); - - /* Free IMB_MGR */ - free_mem(ptr); -} diff --git a/lib/aarch64/clear_regs_mem_aarch64.h b/lib/aarch64/clear_regs_mem_aarch64.h index 1b9587d12965828bada0b7af0a74aeecf2fcd398..39e9aa744e016c692683faa7ab706bb22e8da661 100644 --- a/lib/aarch64/clear_regs_mem_aarch64.h +++ b/lib/aarch64/clear_regs_mem_aarch64.h @@ -35,7 +35,7 @@ #define CLEAR_SCRATCH_GPS() \ do {\ - asm volatile(\ + __asm__ volatile(\ GPR_EOR_SELF(x0) \ GPR_EOR_SELF(x1) \ GPR_EOR_SELF(x2) \ @@ -64,7 +64,7 @@ do {\ #define CLEAR_SCRATCH_SIMD_REGS() \ do{\ - asm volatile(\ + __asm__ volatile(\ SIMD_EOR_SELF(v0) \ SIMD_EOR_SELF(v1) \ SIMD_EOR_SELF(v2) \ diff --git a/lib/aarch64/cpu_features_aarch64.c b/lib/aarch64/cpu_features_aarch64.c index 1f1191d6c82cdde371aa608548b175823a850d0e..56fe4314760b9b9352e8182c1ea0957cbdcf4a08 100644 --- a/lib/aarch64/cpu_features_aarch64.c +++ b/lib/aarch64/cpu_features_aarch64.c @@ -51,12 +51,11 @@ static uint32_t detect_sve(void) return getauxval(AT_HWCAP) & HWCAP_SVE; } -uint64_t cpu_feature_detect(void) +uint64_t __attribute__ ((__target__ ("+sve"))) cpu_feature_detect(void) { uint64_t features = 0; -#ifdef __aarch64__ features |= IMB_FEATURE_AARCH64; -#endif + if (detect_asimd()) { features |= IMB_FEATURE_ASIMD; if (detect_aes()) diff --git a/lib/aarch64/crc32_aarch64.c b/lib/aarch64/crc32_aarch64.c new file mode 100644 index 0000000000000000000000000000000000000000..491c83014d9d11af924a86269c5da7d9b7b64f08 --- /dev/null +++ b/lib/aarch64/crc32_aarch64.c @@ -0,0 +1,127 @@ +/********************************************************************** + Copyright(c) 2023 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ +#include +#include +#include +#include +#include +#define crc32_u8(crc, in) __crc32b(crc, in) +#define crc32_u16(crc, in) __crc32h(crc, in) +#define crc32_u32(crc, in) __crc32w(crc, in) +#define crc32_u64(crc, in) __crc32d(crc, in) +uint32_t crc32_aarch64(const uint8_t* in, size_t size, uint32_t crc); +uint32_t crc32_wimax_ofdma_data_aarch64(const uint8_t* in, size_t size); + +const uint32_t blk_size=2048; +uint32_t crc32_aarch64(const uint8_t* in, size_t size, uint32_t crc) +{ + const uint64_t *in64; + + crc = ~crc; + + // aliagn to 8B + if (((uintptr_t)(in) & 1) && size >= 1) { + crc = crc32_u8(crc, *in); + ++in; + --size; + } + if (((uintptr_t)(in) & 3) && size >= 2) { + crc = crc32_u16(crc, *(const uint16_t*)(in)); + in += 2; + size -= 2; + } + if (((uintptr_t)(in) & 7) && size >= 4) { + crc = crc32_u32(crc, *(const uint32_t*)(in)); + in += 4; + size -= 4; + } + in64= (const uint64_t *)in; + while (size >= 64) { + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + in+=64; + size -= 64; + } + + while (size >= 16) { + crc = crc32_u64(crc, *in64++); + crc = crc32_u64(crc, *in64++); + in += 16; + size -= 16; + } + + if(size >= 8) { + crc = crc32_u64(crc, *(const uint64_t*)(in)); + in += 8; + size -= 8; + } + //~ in64= (const uint64_t *)in; + if (size >= 4) { + crc = crc32_u32(crc, *(const uint32_t*)(in)); + in += 4; + size -= 4; + } + + if (size >= 2) { + crc = crc32_u16(crc, *(const uint16_t*)(in)); + in += 2; + size -= 2; + } + + if (size >= 1) { + crc = crc32_u8(crc, *in); + } + + return ~crc; +} + +uint64_t crc32_wimax_ofdma_data_const[] = { + 0x0000000088fe2237, 0x00000000cbcf3bcb, + 0x00000000567fddeb, 0x0000000010bd4d7c, + 0x000000003a06a4c6, 0x000000002ecc3300, + 0x000000001d49ada7, 0x000000007606eeeb, + 0x00000000f91a84e2, 0x00000000e2ca9d03, + 0x00000000e6228b11, 0x000000008833794c, + 0x000000008c3828a8, 0x0000000064bf7a9b, + 0x0000000075be46b7, 0x00000000569700e5, + 0x00000000e8a45605, 0x00000000c5b9cd4c, + 0x0000000000000000, 0x0000000000000000, + 0xf200aa6600000000, 0x490d678d00000000, + 0x0000000104d101df, 0x0000000104c11db7}; + +uint32_t crc32_wimax_ofdma_data_aarch64(const uint8_t* in, size_t size) { + uint32_t crc = (uint32_t)crc32_wimax_ofdma_data_const[0]; + return crc32_aarch64(in, size, crc); +} diff --git a/lib/aarch64/mb_mgr_aarch64.c b/lib/aarch64/mb_mgr_aarch64.c index 99b89eed590fe2be308ecffa48c3ba3b45610bc5..5efe2018d003711df5c0bb8503cf9fb796310fee 100644 --- a/lib/aarch64/mb_mgr_aarch64.c +++ b/lib/aarch64/mb_mgr_aarch64.c @@ -39,29 +39,53 @@ #include "clear_regs_mem_aarch64.h" #include "include/noaesni.h" #include "include/ipsec_ooo_mgr.h" +#include "arch_aarch64.h" + +#include "include/ooo_mgr_reset.h" + +#define SUBMIT_JOB submit_job_aarch64 +#define FLUSH_JOB flush_job_aarch64 +#define QUEUE_SIZE queue_size_aarch64 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64 +#define GET_NEXT_JOB get_next_job_aarch64 +#define GET_COMPLETED_JOB get_completed_job_aarch64 +#define GET_NEXT_BURST get_next_burst_aarch64 +#define SUBMIT_BURST submit_burst_aarch64 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64 +#define FLUSH_BURST flush_burst_aarch64 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64 +#define SUBMIT_HASH_BURST submit_hash_burst_aarch64 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64 +#define SET_SUITE_ID_FN set_suite_id_aarch64 + +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64 +#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64 +#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64 +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64 + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); -#define SUBMIT_JOB submit_job_aarch64 -#define FLUSH_JOB flush_job_aarch64 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64 -#define GET_NEXT_JOB get_next_job_aarch64 -#define GET_COMPLETED_JOB get_completed_job_aarch64 - -#define QUEUE_SIZE queue_size_aarch64 - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_common -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_common -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64_common -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64_common -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64_common -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64_common -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64_common -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64_common -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_common -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_common -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_common -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_common - -static void reset_ooo_mgrs(IMB_MGR *state); + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); +} IMB_DLL_LOCAL void init_mb_mgr_aarch64_internal(IMB_MGR *state, const int reset_mgrs) @@ -102,6 +126,15 @@ init_mb_mgr_aarch64_internal(IMB_MGR *state, const int reset_mgrs) state->get_completed_job = get_completed_job_aarch64; state->flush_job = flush_job_aarch64; state->queue_size = queue_size_aarch64; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck = SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->eea3_1_buffer = zuc_eea3_1_buffer_aarch64; state->eea3_4_buffer = zuc_eea3_4_buffer_aarch64; @@ -125,6 +158,8 @@ init_mb_mgr_aarch64_internal(IMB_MGR *state, const int reset_mgrs) state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_aarch64; state->snow3g_init_key_sched = snow3g_init_key_sched_aarch64; state->snow3g_key_sched_size = snow3g_key_sched_size_aarch64; + + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_aarch64; } void @@ -132,4 +167,4 @@ init_mb_mgr_aarch64(IMB_MGR *state) { init_mb_mgr_aarch64_internal(state, 1); } -#include "mb_mgr_code_aarch64.h" +#include "mb_mgr_code.h" diff --git a/lib/aarch64/mb_mgr_aarch64_no_aesni.c b/lib/aarch64/mb_mgr_aarch64_no_aesni.c index 40eaf5ff5997f5571892914699a2f7c2109a51d3..df4932c33ff33ad4b62080bbe7192fa463687865 100644 --- a/lib/aarch64/mb_mgr_aarch64_no_aesni.c +++ b/lib/aarch64/mb_mgr_aarch64_no_aesni.c @@ -37,29 +37,53 @@ #include "include/noaesni.h" #include "include/error.h" #include "include/ipsec_ooo_mgr.h" +#include "arch_aarch64_noaesni.h" + +#include "include/ooo_mgr_reset.h" + +#define SUBMIT_JOB submit_job_aarch64_no_aesni +#define FLUSH_JOB flush_job_aarch64_no_aesni +#define QUEUE_SIZE queue_size_aarch64_no_aesni +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_no_aesni +#define GET_NEXT_JOB get_next_job_aarch64_no_aesni +#define GET_COMPLETED_JOB get_completed_job_aarch64_no_aesni +#define GET_NEXT_BURST get_next_burst_aarch64_no_aesni +#define SUBMIT_BURST submit_burst_aarch64_no_aesni +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64_no_aesni +#define FLUSH_BURST flush_burst_aarch64_no_aesni +#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64_no_aesni +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64_no_aesni +#define SUBMIT_HASH_BURST submit_hash_burst_aarch64_no_aesni +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_no_aesni +#define SET_SUITE_ID_FN set_suite_id_aarch64_no_aesni + +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_no_aesni +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_no_aesni +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64_no_aesni +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64_no_aesni +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64_no_aesni +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64_no_aesni +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64_no_aesni +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64_no_aesni +#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_no_aesni +#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_no_aesni +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_no_aesni +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_no_aesni + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); -#define SUBMIT_JOB submit_job_aarch64_no_aesni -#define FLUSH_JOB flush_job_aarch64_no_aesni -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_no_aesni -#define GET_NEXT_JOB get_next_job_aarch64_no_aesni -#define GET_COMPLETED_JOB get_completed_job_aarch64_no_aesni - -#define QUEUE_SIZE queue_size_aarch64_no_aesni - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_no_aesni -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_no_aesni -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64_no_aesni -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64_no_aesni -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64_no_aesni -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64_no_aesni -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64_no_aesni -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64_no_aesni -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_no_aesni -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_no_aesni -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_no_aesni -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_no_aesni - -static void reset_ooo_mgrs(IMB_MGR *state); + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); +} IMB_DLL_LOCAL void init_mb_mgr_aarch64_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) @@ -85,38 +109,49 @@ init_mb_mgr_aarch64_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) } /* set AARCH64 NO AESNI handlers */ - state->get_next_job = get_next_job_aarch64_no_aesni; - state->submit_job = submit_job_aarch64_no_aesni; - state->submit_job_nocheck = submit_job_nocheck_aarch64_no_aesni; - state->get_completed_job = get_completed_job_aarch64_no_aesni; - state->flush_job = flush_job_aarch64_no_aesni; - state->queue_size = queue_size_aarch64_no_aesni; - - state->eea3_1_buffer = zuc_eea3_1_buffer_aarch64_no_aesni; - state->eea3_4_buffer = zuc_eea3_4_buffer_aarch64_no_aesni; - state->eea3_n_buffer = zuc_eea3_n_buffer_aarch64_no_aesni; - state->zuc256_eea3_1_buffer = zuc256_eea3_1_buffer_aarch64_no_aesni; - state->zuc256_eea3_n_buffer = zuc256_eea3_n_buffer_aarch64_no_aesni; - state->eia3_1_buffer = zuc_eia3_1_buffer_aarch64_no_aesni; - state->eia3_n_buffer = zuc_eia3_n_buffer_aarch64_no_aesni; - state->zuc256_eia3_1_buffer = zuc256_eia3_1_buffer_aarch64_no_aesni; - state->zuc256_eia3_n_buffer = zuc256_eia3_n_buffer_aarch64_no_aesni; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_aarch64_no_aesni; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_aarch64_no_aesni; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_aarch64_no_aesni; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_aarch64_no_aesni; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_aarch64_no_aesni; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_aarch64_no_aesni; + state->get_next_job = get_next_job_aarch64_no_aesni; + state->submit_job = submit_job_aarch64_no_aesni; + state->submit_job_nocheck = submit_job_nocheck_aarch64_no_aesni; + state->get_completed_job = get_completed_job_aarch64_no_aesni; + state->flush_job = flush_job_aarch64_no_aesni; + state->queue_size = queue_size_aarch64_no_aesni; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck = SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; + + state->eea3_1_buffer = zuc_eea3_1_buffer_aarch64_no_aesni; + state->eea3_4_buffer = zuc_eea3_4_buffer_aarch64_no_aesni; + state->eea3_n_buffer = zuc_eea3_n_buffer_aarch64_no_aesni; + state->zuc256_eea3_1_buffer = zuc256_eea3_1_buffer_aarch64_no_aesni; + state->zuc256_eea3_n_buffer = zuc256_eea3_n_buffer_aarch64_no_aesni; + state->eia3_1_buffer = zuc_eia3_1_buffer_aarch64_no_aesni; + state->eia3_n_buffer = zuc_eia3_n_buffer_aarch64_no_aesni; + state->zuc256_eia3_1_buffer = zuc256_eia3_1_buffer_aarch64_no_aesni; + state->zuc256_eia3_n_buffer = zuc256_eia3_n_buffer_aarch64_no_aesni; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_aarch64_no_aesni; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_aarch64_no_aesni; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_aarch64_no_aesni; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_aarch64_no_aesni; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_aarch64_no_aesni; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_aarch64_no_aesni; state->snow3g_f8_4_buffer_multikey = snow3g_f8_4_buffer_multikey_aarch64_no_aesni; state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_aarch64_no_aesni; state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_aarch64_no_aesni; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_aarch64_no_aesni; - state->snow3g_init_key_sched = snow3g_init_key_sched_aarch64_no_aesni; - state->snow3g_key_sched_size = snow3g_key_sched_size_aarch64_no_aesni; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_aarch64_no_aesni; + state->snow3g_init_key_sched = snow3g_init_key_sched_aarch64_no_aesni; + state->snow3g_key_sched_size = snow3g_key_sched_size_aarch64_no_aesni; + + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_aarch64; } void @@ -124,4 +159,4 @@ init_mb_mgr_aarch64_no_aesni(IMB_MGR *state) { init_mb_mgr_aarch64_no_aesni_internal(state, 1); } -#include "mb_mgr_code_aarch64.h" +#include "mb_mgr_code.h" diff --git a/lib/aarch64/mb_mgr_aarch64_sve256.c b/lib/aarch64/mb_mgr_aarch64_sve256.c index e8bced59e09c1ecb437e04c270f87dcd8ea0e69e..391540773e8b749a42f7afca770a04857e171809 100644 --- a/lib/aarch64/mb_mgr_aarch64_sve256.c +++ b/lib/aarch64/mb_mgr_aarch64_sve256.c @@ -39,29 +39,54 @@ #include "clear_regs_mem_aarch64.h" #include "include/noaesni.h" #include "include/ipsec_ooo_mgr.h" +#include "arch_aarch64_sve256.h" +#include "arch_aarch64.h" + +#include "include/ooo_mgr_reset.h" + +#define SUBMIT_JOB submit_job_aarch64_sve256 +#define FLUSH_JOB flush_job_aarch64_sve256 +#define QUEUE_SIZE queue_size_aarch64_sve256 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_sve256 +#define GET_NEXT_JOB get_next_job_aarch64_sve256 +#define GET_COMPLETED_JOB get_completed_job_aarch64_sve256 +#define GET_NEXT_BURST get_next_burst_aarch64_sve256 +#define SUBMIT_BURST submit_burst_aarch64_sve256 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_aarch64_sve256 +#define FLUSH_BURST flush_burst_aarch64_sve256 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_aarch64_sve256 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_aarch64_sve256 +#define SUBMIT_HASH_BURST submit_hash_burst_aarch64_sve256 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_aarch64_sve256 +#define SET_SUITE_ID_FN set_suite_id_aarch64_sve256 + +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64 +#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_sve256 +#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_sve256 +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_sve256 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_sve256 + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); -#define SUBMIT_JOB submit_job_aarch64_sve256 -#define FLUSH_JOB flush_job_aarch64_sve256 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_aarch64_sve256 -#define GET_NEXT_JOB get_next_job_aarch64_sve256 -#define GET_COMPLETED_JOB get_completed_job_aarch64_sve256 - -#define QUEUE_SIZE queue_size_aarch64_sve256 - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_aarch64_common -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_aarch64_common -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_aarch64_common -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_aarch64_common -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64_common -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64_common -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64_common -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64_common -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_sve256 -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_sve256 -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_sve256 -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_sve256 - -static void reset_ooo_mgrs(IMB_MGR *state); + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); +} IMB_DLL_LOCAL void init_mb_mgr_aarch64_sve256_internal(IMB_MGR *state, const int reset_mgrs) @@ -102,6 +127,15 @@ init_mb_mgr_aarch64_sve256_internal(IMB_MGR *state, const int reset_mgrs) state->get_completed_job = get_completed_job_aarch64_sve256; state->flush_job = flush_job_aarch64_sve256; state->queue_size = queue_size_aarch64_sve256; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck = SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->eea3_1_buffer = zuc_eea3_1_buffer_aarch64; state->eea3_4_buffer = zuc_eea3_4_buffer_aarch64; @@ -125,6 +159,8 @@ init_mb_mgr_aarch64_sve256_internal(IMB_MGR *state, const int reset_mgrs) state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_aarch64_sve256; state->snow3g_init_key_sched = snow3g_init_key_sched_aarch64_sve256; state->snow3g_key_sched_size = snow3g_key_sched_size_aarch64_sve256; + + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_aarch64; } void @@ -133,4 +169,4 @@ init_mb_mgr_aarch64_sve256(IMB_MGR *state) IMB_ASSERT(state->features & IMB_FEATURE_SVE256); init_mb_mgr_aarch64_sve256_internal(state, 1); } -#include "mb_mgr_code_aarch64.h" +#include "mb_mgr_code.h" diff --git a/lib/aarch64/mb_mgr_code_aarch64.h b/lib/aarch64/mb_mgr_code_aarch64.h deleted file mode 100644 index 4db3491caf5dede56a8faa66540d23455c4b87a6..0000000000000000000000000000000000000000 --- a/lib/aarch64/mb_mgr_code_aarch64.h +++ /dev/null @@ -1,888 +0,0 @@ -/********************************************************************** - Copyright(c) 2021-2022 Arm Corporation All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Arm Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -**********************************************************************/ -#ifndef MB_MGR_CODE_H -#define MB_MGR_CODE_H - -/* - * This contains the bulk of the mb_mgr code, with #define's to build - * an AARCH64 version (see mb_mgr_aarch64.c). - * - * get_next_job() returns a job object. This must be filled in and returned - * via submit_job() before get_next_job() is called again. - * - * submit_job() and flush_job() returns a job object. This job object ceases - * to be usable at the next call to get_next_job() - */ - -#include /* memcpy(), memset() */ - -#include "clear_regs_mem_aarch64.h" -#include "ipsec-mb.h" -#include "error.h" - -#define BSWAP64 __builtin_bswap64 - -IMB_JOB *SUBMIT_JOB_ZUC_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job); -IMB_JOB *SUBMIT_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job); -IMB_JOB *FLUSH_JOB_ZUC_EEA3(MB_MGR_ZUC_OOO *state); -IMB_JOB *FLUSH_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state); -IMB_JOB *SUBMIT_JOB_ZUC_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job); -IMB_JOB *SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job); -IMB_JOB *FLUSH_JOB_ZUC_EIA3(MB_MGR_ZUC_OOO *state); -IMB_JOB *FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state); -IMB_JOB *SUBMIT_JOB_SNOW3G_UEA2(IMB_MGR *state, IMB_JOB *job); -IMB_JOB *FLUSH_JOB_SNOW3G_UEA2(IMB_MGR *state); -IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(IMB_MGR *state, IMB_JOB *job); -IMB_JOB *FLUSH_JOB_SNOW3G_UIA2(IMB_MGR *state); - -static void -reset_ooo_mgrs(IMB_MGR *state) -{ - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; - - /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFF03020100; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFF03020100; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFF03020100; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; - - /* Init SNOW3G out-of-order fields */ - memset(snow3g_uea2_ooo->lens, 0, - sizeof(snow3g_uea2_ooo->lens)); - memset(snow3g_uea2_ooo->job_in_lane, 0, - sizeof(snow3g_uea2_ooo->job_in_lane)); - memset(snow3g_uea2_ooo->bits_fixup, 0, - sizeof(snow3g_uea2_ooo->bits_fixup)); - memset(&(snow3g_uea2_ooo->args), 0, - sizeof(snow3g_uea2_ooo->args)); - snow3g_uea2_ooo->init_mask = 0; - // each 4 bit indicate one lane, at most 16 buffer - snow3g_uea2_ooo->unused_lanes = 0xFEDCBA9876543210; - snow3g_uea2_ooo->num_lanes_inuse = 0; - snow3g_uea2_ooo->init_done = 0; - memset(snow3g_uea2_ooo->ks, 0, - sizeof(snow3g_uea2_ooo->ks)); - - memset(snow3g_uia2_ooo->lens, 0, - sizeof(snow3g_uia2_ooo->lens)); - memset(snow3g_uia2_ooo->job_in_lane, 0, - sizeof(snow3g_uia2_ooo->job_in_lane)); - memset(snow3g_uia2_ooo->bits_fixup, 0, - sizeof(snow3g_uia2_ooo->bits_fixup)); - memset(&(snow3g_uia2_ooo->args), 0, - sizeof(snow3g_uia2_ooo->args)); - snow3g_uia2_ooo->init_mask = 0; - // each 4 bit indicate one lane, at most 16 buffer - snow3g_uia2_ooo->unused_lanes = 0xFEDCBA9876543210; - snow3g_uia2_ooo->num_lanes_inuse = 0; - snow3g_uia2_ooo->init_done = 0; - memset(snow3g_uia2_ooo->ks, 0, - sizeof(snow3g_uia2_ooo->ks)); - return; -} - -/* - * JOBS() and ADV_JOBS() moved into mb_mgr_code.h - * get_next_job() and get_completed_job() API's are no longer inlines. - * For binary compatibility they have been made proper symbols. - */ -__forceinline -IMB_JOB *JOBS(IMB_MGR *state, const int offset) -{ - char *cp = (char *)state->jobs; - - return (IMB_JOB *)(cp + offset); -} - -__forceinline -void ADV_JOBS(int *ptr) -{ - *ptr += sizeof(IMB_JOB); - if (*ptr >= (int) (IMB_MAX_JOBS * sizeof(IMB_JOB))) - *ptr = 0; -} - -__forceinline -IMB_JOB * -submit_snow3g_uea2_job(IMB_MGR *state, IMB_JOB *job) -{ - const snow3g_key_schedule_t *key = job->enc_keys; - const uint32_t msg_bitlen = - (const uint32_t)job->msg_len_to_cipher_in_bits; - const uint32_t msg_bitoff = - (const uint32_t)job->cipher_start_src_offset_in_bits; - - /* Use bit length API if - * - msg length is not a multiple of bytes - * - bit offset is not a multiple of bytes - */ - if ((msg_bitlen & 0x07) || (msg_bitoff & 0x07)) { - IMB_SNOW3G_F8_1_BUFFER_BIT(state, key, job->iv, job->src, - job->dst, msg_bitlen, msg_bitoff); - } else { - const uint32_t msg_bytelen = msg_bitlen >> 3; - const uint32_t msg_byteoff = msg_bitoff >> 3; - const void *src = job->src + msg_byteoff; - void *dst = job->dst + msg_byteoff; - - IMB_SNOW3G_F8_1_BUFFER(state, key, job->iv, src, - dst, msg_bytelen); - } - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -submit_zuc_eea3_job(IMB_MGR *state, IMB_JOB *job) -{ - const uint8_t *pKeys = (const uint8_t *)job->enc_keys; - const uint8_t *pIvs = job->iv; - const uint8_t *pSrc = job->src; - uint8_t *pDst = job->dst; - uint32_t byteLength = job->msg_len_to_cipher_in_bytes; - uint64_t ivLength = job->iv_len_in_bytes; - - if (16 == job->key_len_in_bytes) { - IMB_ZUC_EEA3_1_BUFFER(state, pKeys, pIvs, pSrc, pDst, byteLength); - } else { - IMB_ZUC256_EEA3_1_BUFFER(state, pKeys, pIvs, ivLength, pSrc, pDst, byteLength); - } - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - - if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_SNOW3G_UEA2(state, job); - } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); - } else { /* assume 32 */ - return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); - } - } else { /* assume IMB_CIPHER_NULL */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } -} - -__forceinline -IMB_JOB * -FLUSH_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - - if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); - } else { /* assume 32 */ - return FLUSH_JOB_ZUC256_EEA3(zuc256_eea3_ooo); - } - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode){ - return FLUSH_JOB_SNOW3G_UEA2(state); - } else { /* assume IMB_CIPHER_NULL */ - return NULL; - } -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - - if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_SNOW3G_UEA2(state, job); - } else if(IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); - } else { /* assume 32 */ - return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); - } - } else { - /* assume IMB_CIPHER_NULL */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } -} - -__forceinline -IMB_JOB * -FLUSH_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - - if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); - } else { /* assume 32 */ - return FLUSH_JOB_ZUC256_EEA3(zuc256_eea3_ooo); - } - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode){ - return FLUSH_JOB_SNOW3G_UEA2(state); - } - (void) state; - return NULL; -} - -__forceinline -IMB_JOB * -submit_zuc_eia3_job(IMB_MGR *state, IMB_JOB *job) -{ - const uint8_t *pKeys = job->u.ZUC_EIA3._key; - const uint8_t *pIvs = job->u.ZUC_EIA3._iv; - const uint8_t *pSrc = job->src; - uint32_t bitLength = job->msg_len_to_hash_in_bits; - uint32_t *pMacI = (uint32_t *)job->auth_tag_output; - uint64_t ivLength = job->iv_len_in_bytes; - uint64_t tagLength = job->auth_tag_output_len_in_bytes; - - if (IMB_AUTH_ZUC_EIA3_BITLEN == job->hash_alg) { - IMB_ZUC_EIA3_1_BUFFER(state, pKeys, pIvs, pSrc, bitLength, pMacI); - } else { - IMB_ZUC256_EIA3_1_BUFFER(state, pKeys, pIvs, ivLength, pSrc, - bitLength, pMacI, tagLength); - } - - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -} - -/* ========================================================================= */ -/* Hash submit & flush functions */ -/* ========================================================================= */ -__forceinline -IMB_JOB * -SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - - switch (job->hash_alg) { - case IMB_AUTH_SNOW3G_UIA2_BITLEN: - return SUBMIT_JOB_SNOW3G_UIA2(state, job); - case IMB_AUTH_ZUC_EIA3_BITLEN: - return SUBMIT_JOB_ZUC_EIA3(zuc_eia3_ooo, job); - case IMB_AUTH_ZUC256_EIA3_BITLEN: - return SUBMIT_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job); - default: - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - } -} - -__forceinline -IMB_JOB * -FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - - switch (job->hash_alg) { - case IMB_AUTH_SNOW3G_UIA2_BITLEN: - return FLUSH_JOB_SNOW3G_UIA2(state); - case IMB_AUTH_ZUC_EIA3_BITLEN: - return FLUSH_JOB_ZUC_EIA3(zuc_eia3_ooo); - case IMB_AUTH_ZUC256_EIA3_BITLEN: - return FLUSH_JOB_ZUC256_EIA3(zuc256_eia3_ooo); - default: - if (!(job->status & IMB_STATUS_COMPLETED_AUTH)) { - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - } - return NULL; - } -} - - -/* ========================================================================= */ -/* Job submit & flush functions */ -/* ========================================================================= */ - -#define SNOW3G_MAX_BITLEN (UINT32_MAX) -#define MB_MAX_LEN16 ((1 << 16) - 2) - -__forceinline int -is_job_invalid(IMB_MGR *state, const IMB_JOB *job) -{ - const uint64_t auth_tag_len_ipsec[] = { - 0, /* INVALID selection */ - 12, /* IMB_AUTH_HMAC_SHA_1 */ - 14, /* IMB_AUTH_HMAC_SHA_224 */ - 16, /* IMB_AUTH_HMAC_SHA_256 */ - 24, /* IMB_AUTH_HMAC_SHA_384 */ - 32, /* IMB_AUTH_HMAC_SHA_512 */ - 12, /* IMB_AUTH_AES_XCBC */ - 12, /* IMB_AUTH_MD5 */ - 0, /* IMB_AUTH_NULL */ - 16, /* IMB_AUTH_AES_GMAC */ - 0, /* IMB_AUTH_CUSTOM */ - 0, /* IMB_AUTH_AES_CCM */ - 16, /* IMB_AUTH_AES_CMAC */ - 20, /* IMB_AUTH_SHA_1 */ - 28, /* IMB_AUTH_SHA_224 */ - 32, /* IMB_AUTH_SHA_256 */ - 48, /* IMB_AUTH_SHA_384 */ - 64, /* IMB_AUTH_SHA_512 */ - 4, /* IMB_AUTH_AES_CMAC 3GPP */ - 8, /* IMB_AUTH_PON_CRC_BIP */ - 4, /* IMB_AUTH_ZUC_EIA3_BITLEN */ - 4, /* IMB_AUTH_DOCSIS_CRC32 */ - 4, /* IMB_AUTH_SNOW3G_UIA2_BITLEN */ - 4, /* IMB_AUTH_KASUMI_UIA1 */ - 16, /* IMB_AUTH_AES_GMAC_128 */ - 16, /* IMB_AUTH_AES_GMAC_192 */ - 16, /* IMB_AUTH_AES_GMAC_256 */ - 16, /* IMB_AUTH_AES_CMAC_256 */ - 16, /* IMB_AUTH_POLY1305 */ - 16, /* IMB_AUTH_CHACHA_POLY1305 */ - 16, /* IMB_AUTH_CHACHA_POLY1305_SGL */ - 4, /* IMB_AUTH_ZUC256_EIA3_BITLEN */ - 16, /* IMB_AUTH_SNOW_V_AEAD */ - 16, /* IMB_AUTH_AES_GCM_SGL */ - 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ - 4, /* IMB_AUTH_CRC32_SCTP */ - 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ - 4, /* IMB_AUTH_CRC24_LTE_A */ - 4, /* IMB_AUTH_CRC24_LTE_B */ - 4, /* IMB_AUTH_CRC16_X25 */ - 4, /* IMB_AUTH_CRC16_FP_DATA */ - 4, /* IMB_AUTH_CRC11_FP_HEADER */ - 4, /* IMB_AUTH_CRC10_IUUP_DATA */ - 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ - 4, /* IMB_AUTH_CRC7_FP_HEADER */ - 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ - }; - - // Todo: is_job_invalid was changed, arm should follow that change - if (job->cipher_direction != IMB_DIR_DECRYPT && - job->cipher_direction != IMB_DIR_ENCRYPT && - job->cipher_mode != IMB_CIPHER_NULL) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_DIR); - return 1; - } - switch (job->cipher_mode) { - case IMB_CIPHER_NULL: - /* - * No checks required for this mode - * @note NULL cipher doesn't perform memory copy operation - * from source to destination - */ - break; - case IMB_CIPHER_SNOW3G_UEA2_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->key_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bits == 0 || - job->msg_len_to_cipher_in_bits > SNOW3G_MAX_BITLEN) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_ZUC_EEA3: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->key_len_in_bytes != UINT64_C(16) && - job->key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes == 0 || - job->msg_len_to_cipher_in_bytes > ZUC_MAX_BYTELEN) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->key_len_in_bytes == UINT64_C(16)) { - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - } else { - if (job->iv_len_in_bytes != UINT64_C(23) && - job->iv_len_in_bytes != UINT64_C(25)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - } - break; - default: - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - - switch (job->hash_alg) { - case IMB_AUTH_NULL: - break; - case IMB_AUTH_SNOW3G_UIA2_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if ((job->msg_len_to_hash_in_bits == 0) || - (job->msg_len_to_hash_in_bits > SNOW3G_MAX_BITLEN)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.SNOW3G_UIA2._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->u.SNOW3G_UIA2._iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->auth_tag_output_len_in_bytes != UINT64_C(4)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_ZUC_EIA3_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if ((job->msg_len_to_hash_in_bits < ZUC_MIN_BITLEN) || - (job->msg_len_to_hash_in_bits > ZUC_MAX_BITLEN)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.ZUC_EIA3._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->u.ZUC_EIA3._iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_ZUC256_EIA3_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if ((job->msg_len_to_hash_in_bits < ZUC_MIN_BITLEN) || - (job->msg_len_to_hash_in_bits > ZUC_MAX_BITLEN)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.ZUC_EIA3._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->u.ZUC_EIA3._iv == NULL) { - /* If 25-byte IV is NULL, check 23-byte IV */ - if (job->u.ZUC_EIA3._iv23 == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - } - if ((job->auth_tag_output_len_in_bytes != 4) && - (job->auth_tag_output_len_in_bytes != 8) && - (job->auth_tag_output_len_in_bytes != 16)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - default: - imb_set_errno(state, IMB_ERR_HASH_ALGO); - return 1; - } - return 0; -} - -__forceinline -IMB_JOB *SUBMIT_JOB_AES(IMB_MGR *state, IMB_JOB *job) -{ - if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = SUBMIT_JOB_AES_ENC(state, job); - else - job = SUBMIT_JOB_AES_DEC(state, job); - - return job; -} - -__forceinline -IMB_JOB *FLUSH_JOB_AES(IMB_MGR *state, IMB_JOB *job) -{ - if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = FLUSH_JOB_AES_ENC(state, job); - else - job = FLUSH_JOB_AES_DEC(state, job); - - return job; -} - -/* submit a half-completed job, based on the status */ -__forceinline -IMB_JOB *RESUBMIT_JOB(IMB_MGR *state, IMB_JOB *job) -{ - while (job != NULL && job->status < IMB_STATUS_COMPLETED) { - if (job->status == IMB_STATUS_COMPLETED_AUTH) - job = SUBMIT_JOB_AES(state, job); - else /* assumed job->status = IMB_STATUS_COMPLETED_CIPHER */ - job = SUBMIT_JOB_HASH(state, job); - } - - return job; -} - -__forceinline -IMB_JOB *submit_new_job(IMB_MGR *state, IMB_JOB *job) -{ - if (job->chain_order == IMB_ORDER_CIPHER_HASH) - job = SUBMIT_JOB_AES(state, job); - else - job = SUBMIT_JOB_HASH(state, job); - - job = RESUBMIT_JOB(state, job); - return job; -} - -__forceinline -void complete_job(IMB_MGR *state, IMB_JOB *job) -{ - if (job->chain_order == IMB_ORDER_CIPHER_HASH) { - /* while() loop optimized for cipher_hash order */ - while (job->status < IMB_STATUS_COMPLETED) { - IMB_JOB *tmp = FLUSH_JOB_AES(state, job); - - if (tmp == NULL) - tmp = FLUSH_JOB_HASH(state, job); - - (void) RESUBMIT_JOB(state, tmp); - } - } else { - /* while() loop optimized for hash_cipher order */ - while (job->status < IMB_STATUS_COMPLETED) { - IMB_JOB *tmp = FLUSH_JOB_HASH(state, job); - - if (tmp == NULL) - tmp = FLUSH_JOB_AES(state, job); - - (void) RESUBMIT_JOB(state, tmp); - } - } -} - -__forceinline -IMB_JOB * -submit_job_and_check(IMB_MGR *state, const int run_check) -{ - IMB_JOB *job = NULL; - - job = JOBS(state, state->next_job); - - if (run_check) { - if (is_job_invalid(state, job)) { - job->status = IMB_STATUS_INVALID_ARGS; - } else { - job->status = IMB_STATUS_BEING_PROCESSED; - job = submit_new_job(state, job); - } - } else { - job->status = IMB_STATUS_BEING_PROCESSED; - job = submit_new_job(state, job); - } - - if (state->earliest_job < 0) { - /* state was previously empty */ - if (job == NULL) - state->earliest_job = state->next_job; - ADV_JOBS(&state->next_job); - goto exit; - } - - ADV_JOBS(&state->next_job); - - if (state->earliest_job == state->next_job) { - /* Full */ - job = JOBS(state, state->earliest_job); - complete_job(state, job); - ADV_JOBS(&state->earliest_job); - goto exit; - } - - /* not full */ - job = JOBS(state, state->earliest_job); - if (job->status < IMB_STATUS_COMPLETED) { - job = NULL; - goto exit; - } - - ADV_JOBS(&state->earliest_job); -exit: -#ifdef SAFE_DATA - CLEAR_SCRATCH_GPS(); - CLEAR_SCRATCH_SIMD_REGS(); -#endif /* SAFE_DATA */ - - return job; -} - -IMB_JOB * -SUBMIT_JOB(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - - return submit_job_and_check(state, 1); -} - -IMB_JOB * -SUBMIT_JOB_NOCHECK(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - - return submit_job_and_check(state, 0); -} - -IMB_JOB * -FLUSH_JOB(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - IMB_JOB *job; - if (state->earliest_job < 0) - return NULL; /* empty */ - - job = JOBS(state, state->earliest_job); - complete_job(state, job); - - ADV_JOBS(&state->earliest_job); - - if (state->earliest_job == state->next_job) - state->earliest_job = -1; /* becomes empty */ - -#ifdef SAFE_DATA - CLEAR_SCRATCH_GPS(); - CLEAR_SCRATCH_SIMD_REGS(); -#endif /* SAFE_DATA */ - - return job; -} - -/* ========================================================================= */ - -uint32_t -QUEUE_SIZE(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return 0; - } -#endif - int a, b; - - if (state->earliest_job < 0) - return 0; - a = state->next_job / sizeof(IMB_JOB); - b = state->earliest_job / sizeof(IMB_JOB); - return ((a-b) & (IMB_MAX_JOBS-1)); -} - -IMB_JOB * -GET_COMPLETED_JOB(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - IMB_JOB *job; - - if (state->earliest_job < 0) - return NULL; - - job = JOBS(state, state->earliest_job); - if (job->status < IMB_STATUS_COMPLETED) - return NULL; - - ADV_JOBS(&state->earliest_job); - - if (state->earliest_job == state->next_job) - state->earliest_job = -1; - - return job; -} - -IMB_JOB * -GET_NEXT_JOB(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - - return JOBS(state, state->next_job); -} - -#endif /* MB_MGR_CODE_H */ diff --git a/lib/aarch64/mb_mgr_snow3g_submit_flush_aarch64.c b/lib/aarch64/mb_mgr_snow3g_submit_flush_aarch64.c index c20e49881dad1a44ceb1a24fa261f99b5ef2f807..4512868f835474f609531b1b38379428dc07e917 100644 --- a/lib/aarch64/mb_mgr_snow3g_submit_flush_aarch64.c +++ b/lib/aarch64/mb_mgr_snow3g_submit_flush_aarch64.c @@ -27,10 +27,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ #ifndef SUBMIT_JOB_SNOW3G_UEA2 -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64_common -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64_common -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64_common -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64_common +#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_aarch64 +#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_aarch64 +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_aarch64 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_aarch64 #define SNOW3G_F8_MULTI_BUFFER_INITIALIZE_JOB snow3g_f8_4_buffer_initialize_aarch64_neon_asm #define SNOW3G_F8_MULTI_BUFFER_STREAM_JOB snow3g_f8_4_buffer_stream_aarch64_neon_asm #define SNOW3G_F9_MULTI_BUFFER_KEYSTREAM_JOB snow3g_f9_4_buffer_keystream_aarch64_neon_asm diff --git a/lib/aarch64/mb_mgr_snow3g_submit_flush_common_aarch64.h b/lib/aarch64/mb_mgr_snow3g_submit_flush_common_aarch64.h index 4a88fed1f92228184ff7b7541471383e6e0bc511..13bca11b32e406ecfde667b2a504aa72a11ca135 100644 --- a/lib/aarch64/mb_mgr_snow3g_submit_flush_common_aarch64.h +++ b/lib/aarch64/mb_mgr_snow3g_submit_flush_common_aarch64.h @@ -61,8 +61,8 @@ IMB_JOB *SUBMIT_JOB_SNOW3G_UEA2(IMB_MGR *state, IMB_JOB *job); IMB_JOB *FLUSH_JOB_SNOW3G_UEA2(IMB_MGR *state); -IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(IMB_MGR *state, IMB_JOB *job); -IMB_JOB *FLUSH_JOB_SNOW3G_UIA2(IMB_MGR *state); +IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); +IMB_JOB *FLUSH_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state); void SNOW3G_F8_1_BUFFER_STREAM_JOB(void *pCtx, const void *pBufferIn, @@ -361,8 +361,8 @@ IMB_JOB *FLUSH_JOB_SNOW3G_UEA2(IMB_MGR *state) return NULL; } -IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(IMB_MGR *state, - IMB_JOB *job) +IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state, + IMB_JOB *job) { #ifdef SAFE_PARAM /* reset error status */ @@ -392,7 +392,7 @@ IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(IMB_MGR *state, return NULL; } #endif - MB_MGR_SNOW3G_OOO *snow3g_state = state->snow3g_uia2_ooo; + MB_MGR_SNOW3G_OOO *snow3g_state = state; IMB_JOB *ret = NULL; @@ -424,10 +424,10 @@ IMB_JOB *SUBMIT_JOB_SNOW3G_UIA2(IMB_MGR *state, return ret; } -IMB_JOB *FLUSH_JOB_SNOW3G_UIA2(IMB_MGR *state) +IMB_JOB *FLUSH_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state) { IMB_JOB *ret = NULL; - MB_MGR_SNOW3G_OOO *snow3g_state = state->snow3g_uia2_ooo; + MB_MGR_SNOW3G_OOO *snow3g_state = state; if (snow3g_state->num_lanes_inuse == 0) { // empty diff --git a/lib/aarch64/mb_mgr_zuc_submit_flush_aarch64.c b/lib/aarch64/mb_mgr_zuc_submit_flush_aarch64.c index 4f4358e04326a52965b7ca01299997f41e5da00a..cadcc754ecfdbf09978da9ee3647c1b5e2d7f780 100644 --- a/lib/aarch64/mb_mgr_zuc_submit_flush_aarch64.c +++ b/lib/aarch64/mb_mgr_zuc_submit_flush_aarch64.c @@ -33,16 +33,17 @@ #ifdef SAFE_PARAM #include "error.h" #endif +#include "clear_regs_mem_aarch64.h" #ifndef SUBMIT_JOB_ZUC128_EEA3 -#define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_aarch64_common -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64_common -#define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_aarch64_common -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64_common -#define SUBMIT_JOB_ZUC128_EIA3 submit_job_zuc_eia3_aarch64_common -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64_common -#define FLUSH_JOB_ZUC128_EIA3 flush_job_zuc_eia3_aarch64_common -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64_common +#define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_aarch64 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_aarch64 +#define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_aarch64 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_aarch64 +#define SUBMIT_JOB_ZUC128_EIA3 submit_job_zuc_eia3_aarch64 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_aarch64 +#define FLUSH_JOB_ZUC128_EIA3 flush_job_zuc_eia3_aarch64 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_aarch64 #define ZUC_EIA3_4_BUFFER zuc_eia3_4_buffer_job_aarch64 #define ZUC256_EIA3_4_BUFFER zuc256_eia3_4_buffer_job_aarch64 #define ZUC128_INIT_4 asm_ZucInitialization_4_aarch64 @@ -65,9 +66,9 @@ IMB_JOB *SUBMIT_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job); IMB_JOB *FLUSH_JOB_ZUC128_EEA3(MB_MGR_ZUC_OOO *state); IMB_JOB *FLUSH_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state); IMB_JOB *SUBMIT_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job); -IMB_JOB *SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job); +IMB_JOB *SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz); IMB_JOB *FLUSH_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state); -IMB_JOB *FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state); +IMB_JOB *FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); typedef enum { ZUC_128, @@ -292,6 +293,9 @@ static IMB_JOB *zuc_mb_mgr_submit_eea3_job(MB_MGR_ZUC_OOO *state, #ifdef SAFE_DATA memset(state_tmp, 0, MAX_ZUC_STATE_SZ * 4); + /* Clear sensitive data in registers */ + CLEAR_SCRATCH_GPS(); + CLEAR_SCRATCH_SIMD_REGS(); #endif return ret; @@ -387,6 +391,10 @@ static IMB_JOB *zuc_mb_mgr_flush_eea3_job(MB_MGR_ZUC_OOO *state, ZUC_TYPE zuc) } memset(state_tmp, 0, MAX_ZUC_STATE_SZ * 4); + + /* Clear sensitive data in registers */ + CLEAR_SCRATCH_GPS(); + CLEAR_SCRATCH_SIMD_REGS(); #endif return ret; @@ -394,7 +402,8 @@ static IMB_JOB *zuc_mb_mgr_flush_eea3_job(MB_MGR_ZUC_OOO *state, ZUC_TYPE zuc) static IMB_JOB *zuc_mb_mgr_submit_eia3_job(MB_MGR_ZUC_OOO *state, IMB_JOB *job, - ZUC_TYPE zuc) + ZUC_TYPE zuc, + const uint64_t tag_sz) { #ifdef SAFE_PARAM @@ -459,7 +468,7 @@ static IMB_JOB *zuc_mb_mgr_submit_eia3_job(MB_MGR_ZUC_OOO *state, (uint32_t **)state->args.out, state->lens, (const void * const *)state->job_in_lane, - job->auth_tag_output_len_in_bytes); + tag_sz); // clear all lengths(function will authenticate all buffers) for (i = 0; i < ZUC_MB_MAX_LANES_SIMD; i++) { @@ -471,7 +480,9 @@ static IMB_JOB *zuc_mb_mgr_submit_eia3_job(MB_MGR_ZUC_OOO *state, return ret; } -static IMB_JOB *zuc_mb_mgr_flush_eia3_job(MB_MGR_ZUC_OOO *state, ZUC_TYPE key) +static IMB_JOB *zuc_mb_mgr_flush_eia3_job(MB_MGR_ZUC_OOO *state, + ZUC_TYPE key, + const uint64_t tag_sz) { IMB_JOB *ret = NULL; uint32_t min_len, i, idx = 0; @@ -524,7 +535,7 @@ static IMB_JOB *zuc_mb_mgr_flush_eia3_job(MB_MGR_ZUC_OOO *state, ZUC_TYPE key) (uint32_t **)state->args.out, state->lens, (const void * const *)state->job_in_lane, - state->job_in_lane[idx]->auth_tag_output_len_in_bytes); + tag_sz); // clear all lengths of valid jobs and set to FFFFFFFF to NULL jobs for (i = 0; i < ZUC_MB_MAX_LANES_SIMD; i++) { @@ -562,20 +573,22 @@ IMB_JOB *FLUSH_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state) IMB_JOB *SUBMIT_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) { - return zuc_mb_mgr_submit_eia3_job(state, job, ZUC_128); + return zuc_mb_mgr_submit_eia3_job(state, job, ZUC_128, 4); } -IMB_JOB *SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) +IMB_JOB *SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz) { - return zuc_mb_mgr_submit_eia3_job(state, job, ZUC_256); + return zuc_mb_mgr_submit_eia3_job(state, job, ZUC_256, tag_sz); } IMB_JOB *FLUSH_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state) { - return zuc_mb_mgr_flush_eia3_job(state, ZUC_128); + return zuc_mb_mgr_flush_eia3_job(state, ZUC_128, 4); } -IMB_JOB *FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state) +IMB_JOB *FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz) { - return zuc_mb_mgr_flush_eia3_job(state, ZUC_256); + return zuc_mb_mgr_flush_eia3_job(state, ZUC_256, tag_sz); } diff --git a/lib/aarch64/zuc_common.S b/lib/aarch64/zuc_common.inc similarity index 99% rename from lib/aarch64/zuc_common.S rename to lib/aarch64/zuc_common.inc index 7ed729bd510833ba9ff624af5a987425ef62e0b9..6fb5b97e2ce9d9a0550fa8a9776bf59e43af9006 100644 --- a/lib/aarch64/zuc_common.S +++ b/lib/aarch64/zuc_common.inc @@ -27,7 +27,7 @@ .arch armv8-a+crypto -#include "zuc_sbox.S" +#include "zuc_sbox.inc" .section .data .align 3 diff --git a/lib/aarch64/zuc_sbox.S b/lib/aarch64/zuc_sbox.inc similarity index 99% rename from lib/aarch64/zuc_sbox.S rename to lib/aarch64/zuc_sbox.inc index f8445bb6b8a8cdbedbef31f92a00f4c5aa38c484..616380f90af53ee08ae853755078bdc97e98624c 100644 --- a/lib/aarch64/zuc_sbox.S +++ b/lib/aarch64/zuc_sbox.inc @@ -28,7 +28,7 @@ #ifndef _ZUC_SOBX_INC_ #define _ZUC_SOBX_INC_ -#include "aarch64/aesni_emu_aarch64.S" +#include "aarch64/aesni_emu_aarch64.inc" .section .data .align 4 diff --git a/lib/aarch64/zuc_simd.S b/lib/aarch64/zuc_simd.S index 2cd39a3f70c4d7f41e5658b585a92928d6308f04..bec6e4fea038dbb0b404467659231d46c7299532 100644 --- a/lib/aarch64/zuc_simd.S +++ b/lib/aarch64/zuc_simd.S @@ -25,8 +25,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *******************************************************************************/ -#include "zuc_sbox.S" -#include "zuc_common.S" +#include "zuc_sbox.inc" +#include "zuc_common.inc" #ifndef ZUC_CIPHER_4 #define ZUC_CIPHER_4 asm_ZucCipher_4_aarch64 @@ -692,31 +692,6 @@ declare_register qFR2, q28 str qFR1, [pState, #OFS_R1] str qFR2, [pState, #OFS_R2] -#ifdef SAFE_DATA - // clear intermediate value - eor v0.16b, v0.16b, v0.16b - eor v1.16b, v1.16b, v1.16b - eor v2.16b, v2.16b, v2.16b - eor v3.16b, v3.16b, v3.16b - eor v4.16b, v4.16b, v4.16b - eor v5.16b, v5.16b, v5.16b - eor v6.16b, v6.16b, v6.16b - eor v7.16b, v7.16b, v7.16b - eor v8.16b, v8.16b, v8.16b - eor v9.16b, v9.16b, v9.16b - eor v10.16b, v10.16b, v10.16b - eor v11.16b, v11.16b, v11.16b - eor v12.16b, v12.16b, v12.16b - eor v13.16b, v13.16b, v13.16b - eor v14.16b, v14.16b, v14.16b - eor v15.16b, v15.16b, v15.16b - eor v24.16b, v24.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - eor v26.16b, v26.16b, v26.16b - eor v27.16b, v27.16b, v27.16b - eor v28.16b, v28.16b, v28.16b -#endif - FUNC_RESTORE ret @@ -973,35 +948,6 @@ end_load: str qFR1, [pState, #OFS_R1] str qFR2, [pState, #OFS_R2] -#ifdef SAFE_DATA - // clear intermediate value - eor v0.16b, v0.16b, v0.16b - eor v1.16b, v1.16b, v1.16b - eor v2.16b, v2.16b, v2.16b - eor v3.16b, v3.16b, v3.16b - eor v4.16b, v4.16b, v4.16b - eor v5.16b, v5.16b, v5.16b - eor v6.16b, v6.16b, v6.16b - eor v7.16b, v7.16b, v7.16b - eor v8.16b, v8.16b, v8.16b - eor v9.16b, v9.16b, v9.16b - eor v10.16b, v10.16b, v10.16b - eor v11.16b, v11.16b, v11.16b - eor v12.16b, v12.16b, v12.16b - eor v13.16b, v13.16b, v13.16b - eor v14.16b, v14.16b, v14.16b - eor v15.16b, v15.16b, v15.16b - eor v24.16b, v24.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - eor v26.16b, v26.16b, v26.16b - eor v27.16b, v27.16b, v27.16b - eor v28.16b, v28.16b, v28.16b - // clear generated key stream - eor v16.16b, v16.16b, v16.16b - eor v17.16b, v17.16b, v17.16b - eor v18.16b, v18.16b, v18.16b - eor v19.16b, v19.16b, v19.16b -#endif FUNC_RESTORE .endm @@ -1376,46 +1322,6 @@ auth_finish_zuc\KEYLEN: stp pIN0, pIN1, [pIn] stp pIN2, pIN3, [pIn, 16] -#ifdef SAFE_DATA - // clear intermediate value - eor v0.16b, v0.16b, v0.16b - eor v1.16b, v1.16b, v1.16b - eor v2.16b, v2.16b, v2.16b - eor v3.16b, v3.16b, v3.16b - eor v4.16b, v4.16b, v4.16b - eor v5.16b, v5.16b, v5.16b - eor v6.16b, v6.16b, v6.16b - eor v7.16b, v7.16b, v7.16b - eor v8.16b, v8.16b, v8.16b - eor v9.16b, v9.16b, v9.16b - eor v10.16b, v10.16b, v10.16b - eor v11.16b, v11.16b, v11.16b - eor v12.16b, v12.16b, v12.16b - eor v13.16b, v13.16b, v13.16b - eor v14.16b, v14.16b, v14.16b - eor v15.16b, v15.16b, v15.16b - eor v24.16b, v24.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - eor v26.16b, v26.16b, v26.16b - eor v27.16b, v27.16b, v27.16b - eor v28.16b, v28.16b, v28.16b - // clear generated key stream - eor v16.16b, v16.16b, v16.16b - eor v17.16b, v17.16b, v17.16b - eor v18.16b, v18.16b, v18.16b - eor v19.16b, v19.16b, v19.16b - eor v20.16b, v20.16b, v20.16b - eor v21.16b, v21.16b, v21.16b - eor v22.16b, v22.16b, v22.16b - eor v23.16b, v23.16b, v23.16b - // clear intermediate digest - eor wT, wT, wT - eor wT0, wT0, wT0 - eor wT1, wT1, wT1 - eor wT2, wT2, wT2 - eor wT3, wT3, wT3 -#endif - add sp, sp, 112 FUNC_RESTORE ret @@ -1517,21 +1423,6 @@ TAG_4B_EMUL_PMULL: b ROUND_END ROUND_END: -#ifdef SAFE_DATA - // clear intermediate value - eor v0.16b, v0.16b, v0.16b - eor v1.16b, v1.16b, v1.16b - eor v2.16b, v2.16b, v2.16b - eor v3.16b, v3.16b, v3.16b - eor v4.16b, v4.16b, v4.16b - eor v5.16b, v5.16b, v5.16b - eor v6.16b, v6.16b, v6.16b - eor v7.16b, v7.16b, v7.16b - eor v16.16b, v16.16b, v16.16b - eor v17.16b, v17.16b, v17.16b - eor v18.16b, v18.16b, v18.16b - eor w3, w3, w3 -#endif FUNC_RESTORE @@ -1632,24 +1523,6 @@ Eia3Rounds_byte_read: Eia3Rounds_byte_loop_end: -#ifdef SAFE_DATA - // clear intermediate value - eor v0.16b, v0.16b, v0.16b - eor v1.16b, v1.16b, v1.16b - eor v2.16b, v2.16b, v2.16b - eor v3.16b, v3.16b, v3.16b - eor v4.16b, v4.16b, v4.16b - eor v5.16b, v5.16b, v5.16b - eor v6.16b, v6.16b, v6.16b - eor v7.16b, v7.16b, v7.16b - eor v16.16b, v16.16b, v16.16b - eor v17.16b, v17.16b, v17.16b - eor v18.16b, v18.16b, v18.16b - eor x3, x3, x3 - eor x4, x4, x4 - eor x5, x5, x5 -#endif - FUNC_RESTORE ret @@ -1858,31 +1731,6 @@ exit_final_rounds: add v2.2d, v2.2d, v0.2d st1 {v1.2d, v2.2d}, [pOut] -#ifdef SAFE_DATA - // clear intermediate value - eor v0.16b, v0.16b, v0.16b - eor v1.16b, v1.16b, v1.16b - eor v2.16b, v2.16b, v2.16b - eor v3.16b, v3.16b, v3.16b - eor v4.16b, v4.16b, v4.16b - eor v5.16b, v5.16b, v5.16b - eor v6.16b, v6.16b, v6.16b - eor v7.16b, v7.16b, v7.16b - eor v8.16b, v8.16b, v8.16b - eor v9.16b, v9.16b, v9.16b - eor v10.16b, v10.16b, v10.16b - eor v11.16b, v11.16b, v11.16b - eor v12.16b, v12.16b, v12.16b - eor v13.16b, v13.16b, v13.16b - eor v14.16b, v14.16b, v14.16b - eor v15.16b, v15.16b, v15.16b - eor v24.16b, v24.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - eor v26.16b, v26.16b, v26.16b - eor v27.16b, v27.16b, v27.16b - eor v28.16b, v28.16b, v28.16b -#endif - FUNC_RESTORE exit_cipher: @@ -1904,11 +1752,6 @@ START_FUNC(ZUC_XORKEYSTREAM16B) eor XKEY.16b, XKEY.16b, XIN.16b st1 {XKEY.16b}, [pOut] -#ifdef SAFE_DATA - eor XKEY.16b, XKEY.16b, XKEY.16b - eor XIN.16b, XIN.16b, XIN.16b -#endif - ret END_FUNC(ZUC_XORKEYSTREAM16B) diff --git a/lib/aarch64/zuc_simd_no_aesni.S b/lib/aarch64/zuc_simd_no_aesni.S index 11e28b1ffb800396a1a036eab0bcecda00585f3d..3b06c5e0081f4e1a1cefc9a2a9a396d27bb8c563 100644 --- a/lib/aarch64/zuc_simd_no_aesni.S +++ b/lib/aarch64/zuc_simd_no_aesni.S @@ -25,7 +25,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *******************************************************************************/ -#include "aarch64/aesni_emu_aarch64.S" +#include "aarch64/aesni_emu_aarch64.inc" #define INTEL_AESNCLAST EMULATE_AESENCLAST #define ZUC_CIPHER_4 asm_ZucCipher_4_aarch64_no_aesni diff --git a/lib/api_doxygen.conf b/lib/api_doxygen.conf index 9c7a10663aec7cd57af63bdf9e9fe38224988e95..dd26d4ed9647be28681d50daa75032a5ff3e33c4 100644 --- a/lib/api_doxygen.conf +++ b/lib/api_doxygen.conf @@ -1,3 +1,39 @@ +# Doxyfile 1.9.7 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). +# +# Note: +# +# Use doxygen to compare the used configuration file with the template +# configuration file: +# doxygen -x [configFile] +# Use doxygen to compare the used configuration file with the template +# configuration file without replacing the environment variables or CMake type +# replacement variables: +# doxygen -x_noenv [configFile] + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. +# The default value is: UTF-8. + DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded by @@ -6,13 +42,13 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "ipsec-mb" +PROJECT_NAME = ipsec-mb # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = +PROJECT_NUMBER = 1.4 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -34,16 +70,28 @@ PROJECT_LOGO = OUTPUT_DIRECTORY = ../docs/doc_api -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 +# sub-directories (in 2 levels) under the output directory of each output format +# and will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes -# performance problems for the file system. +# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to +# control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO +# Controls the number of sub-directories that will be created when +# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every +# level increment doubles the number of directories, resulting in 4096 +# directories at level 8 which is the default and also the maximum value. The +# sub-directories are organized in 2 levels, the first level always has a fixed +# number of 16 directories. +# Minimum value: 0, maximum value: 8, default value: 8. +# This tag requires that the tag CREATE_SUBDIRS is set to YES. + +CREATE_SUBDIRS_LEVEL = 8 + # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode @@ -55,14 +103,14 @@ ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, +# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English +# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, +# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with +# English messages), Korean, Korean-en (Korean with English messages), Latvian, +# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, +# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, +# Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English @@ -153,6 +201,16 @@ SHORT_NAMES = NO JAVADOC_AUTOBRIEF = NO +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO + # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If # set to NO, the Qt-style will behave just like regular Qt-style comments (thus @@ -173,6 +231,14 @@ QT_AUTOBRIEF = NO MULTILINE_CPP_IS_BRIEF = NO +# By default Python docstrings are displayed as preformatted text and doxygen's +# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the +# doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as doxygen documentation. +# The default value is: YES. + +PYTHON_DOCSTRING = YES + # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. @@ -196,11 +262,16 @@ TAB_SIZE = 8 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines. +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = @@ -232,28 +303,40 @@ OPTIMIZE_FOR_FORTRAN = NO OPTIMIZE_OUTPUT_VHDL = NO +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: -# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: -# Fortran. In the later case the parser tries to guess whether the code is fixed -# or free formatted code, this is the default for Fortran type files), VHDL. For -# instance to make doxygen treat .inc files as Fortran files (default is PHP), -# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files). For instance to make doxygen treat .inc files +# as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C. # # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. +# documentation. See https://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. @@ -261,6 +344,26 @@ EXTENSION_MAPPING = MARKDOWN_SUPPORT = YES +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 5. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 5 + +# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to +# generate identifiers for the Markdown headings. Note: Every identifier is +# unique. +# Possible values are: DOXYGEN Use a fixed 'autotoc_md' string followed by a +# sequence number starting at 0. and GITHUB Use the lower case version of title +# with any whitespace replaced by '-' and punctations characters removed.. +# The default value is: DOXYGEN. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +MARKDOWN_ID_STYLE = DOXYGEN + # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or @@ -286,7 +389,7 @@ BUILTIN_STL_SUPPORT = NO CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. @@ -372,6 +475,27 @@ TYPEDEF_HIDES_STRUCT = NO LOOKUP_CACHE_SIZE = 0 +# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use +# during processing. When set to 0 doxygen will based this on the number of +# cores available in the system. You can set it explicitly to a value larger +# than 0 to get more control over the balance between CPU load and processing +# speed. At this moment only the input processing can be done using multiple +# threads. Since this is still an experimental feature the default is set to 1, +# which effectively disables parallel processing. Please report any issues you +# encounter. Generating dot graphs in parallel is controlled by the +# DOT_NUM_THREADS setting. +# Minimum value: 0, maximum value: 32, default value: 1. + +NUM_PROC_THREADS = 1 + +# If the TIMESTAMP tag is set different from NO then each generated page will +# contain the date or date and time when the page was generated. Setting this to +# NO can help when comparing the output of multiple runs. +# Possible values are: YES, NO, DATETIME and DATE. +# The default value is: NO. + +TIMESTAMP = NO + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- @@ -392,6 +516,12 @@ EXTRACT_ALL = YES EXTRACT_PRIVATE = NO +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal # scope will be included in the documentation. # The default value is: NO. @@ -429,6 +559,13 @@ EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation @@ -440,14 +577,15 @@ HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. +# will also hide undocumented C++ concepts if enabled. This option has no effect +# if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. +# declarations. If set to NO, these declarations will be included in the +# documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO @@ -466,12 +604,20 @@ HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. -# The default value is: system dependent. +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# Possible values are: SYSTEM, NO and YES. +# The default value is: SYSTEM. CASE_SENSE_NAMES = YES @@ -489,6 +635,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -646,7 +798,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -657,7 +810,7 @@ LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. See also \cite for info how to create references. @@ -692,23 +845,50 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = NO +# If WARN_IF_UNDOC_ENUM_VAL option is set to YES, doxygen will warn about +# undocumented enumeration values. If set to NO, doxygen will accept +# undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: NO. + +WARN_IF_UNDOC_ENUM_VAL = NO + # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then doxygen behaves +# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined doxygen will not +# write the warning messages in between other messages but write them at the end +# of a run, in case a WARN_LOGFILE is defined the warning messages will be +# besides being in the defined file also be shown at the end of a run, unless +# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case +# the behavior will remain as with the setting FAIL_ON_WARNINGS. +# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = NO @@ -719,13 +899,27 @@ WARN_AS_ERROR = NO # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) +# See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" +# In the $text part of the WARN_FORMAT command it is possible that a reference +# to a more specific place is given. To make it easier to jump to this place +# (outside of doxygen) the user can define a custom "cut" / "paste" string. +# Example: +# WARN_LINE_FORMAT = "'vi $file +$line'" +# See also: WARN_FORMAT +# The default value is: at line $line of file $file. + +WARN_LINE_FORMAT = "at line $line of file $file" + # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard -# error (stderr). +# error (stderr). In case the file specified cannot be opened for writing the +# warning and error messages are written to standard error. When as file - is +# specified the warning and error messages are written to standard output +# (stdout). WARN_LOGFILE = @@ -739,17 +933,29 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ./ipsec-mb.h ../README.md +INPUT = ./ipsec-mb.h \ + ../README.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: http://www.gnu.org/software/libiconv) for the list of -# possible encodings. +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# See also: INPUT_FILE_ENCODING # The default value is: UTF-8. INPUT_ENCODING = UTF-8 +# This tag can be used to specify the character encoding of the source files +# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify +# character encoding on a per file pattern basis. Doxygen will compare the file +# name with each pattern and apply the encoding instead of the default +# INPUT_ENCODING) if there is a match. The character encodings are a list of the +# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding +# "INPUT_ENCODING" for further information on supported encodings. + +INPUT_FILE_ENCODING = + # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. @@ -758,11 +964,15 @@ INPUT_ENCODING = UTF-8 # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by doxygen. # +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl, -# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js. +# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, +# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C +# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = @@ -801,10 +1011,7 @@ EXCLUDE_PATTERNS = # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* +# ANamespace::AClass, ANamespace::*Test EXCLUDE_SYMBOLS = @@ -849,6 +1056,11 @@ IMAGE_PATH = # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # +# Note that doxygen will use the data processed and written to standard output +# for further processing, therefore nothing else, like debug statements or used +# commands (so in case of a Windows batch file always use @echo OFF), should be +# written to standard output. +# # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by doxygen. @@ -890,6 +1102,15 @@ FILTER_SOURCE_PATTERNS = USE_MDFILE_AS_MAINPAGE = README.md +# The Fortran standard specifies that for fixed formatted Fortran code all +# characters from position 72 are to be considered as comment. A common +# extension is to allow longer lines before the automatic comment starts. The +# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can +# be processed before the automatic comment starts. +# Minimum value: 7, maximum value: 10000, default value: 72. + +FORTRAN_COMMENT_AFTER = 72 + #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- @@ -917,7 +1138,7 @@ INLINE_SOURCES = NO STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# function all documented functions referencing it will be listed. +# entity all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO @@ -949,12 +1170,12 @@ SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system -# (see http://www.gnu.org/software/global/global.html). You will need version +# (see https://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # @@ -976,6 +1197,46 @@ USE_HTAGS = NO VERBATIM_HEADERS = YES +# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the +# clang parser (see: +# http://clang.llvm.org/) for more accurate parsing at the cost of reduced +# performance. This can be particularly helpful with template rich C++ code for +# which doxygen's built-in parser lacks the necessary type information. +# Note: The availability of this option depends on whether or not doxygen was +# generated with the -Duse_libclang=ON option for CMake. +# The default value is: NO. + +CLANG_ASSISTED_PARSING = NO + +# If the CLANG_ASSISTED_PARSING tag is set to YES and the CLANG_ADD_INC_PATHS +# tag is set to YES then doxygen will add the directory of each input to the +# include path. +# The default value is: YES. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. + +CLANG_ADD_INC_PATHS = YES + +# If clang assisted parsing is enabled you can provide the compiler with command +# line options that you would normally use when invoking the compiler. Note that +# the include paths will already be set by doxygen for the files and directories +# specified with INPUT and INCLUDE_PATH. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. + +CLANG_OPTIONS = + +# If clang assisted parsing is enabled you can provide the clang parser with the +# path to the directory containing a file called compile_commands.json. This +# file is the compilation database (see: +# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) containing the +# options used when the source files were built. This is equivalent to +# specifying the -p option to a clang tool, such as clang-check. These options +# will then be passed to the parser. Any options specified with CLANG_OPTIONS +# will be added as well. +# Note: The availability of this option depends on whether or not doxygen was +# generated with the -Duse_libclang=ON option for CMake. + +CLANG_DATABASE_PATH = + #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- @@ -987,10 +1248,11 @@ VERBATIM_HEADERS = YES ALPHABETICAL_INDEX = YES -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. +# The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes) +# that should be ignored while generating the index headers. The IGNORE_PREFIX +# tag works for classes, function and member names. The entity will be placed in +# the alphabetical list under the first letter of the entity name that remains +# after removing the prefix. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = @@ -1069,7 +1331,12 @@ HTML_STYLESHEET = # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. +# list). +# Note: Since the styling of scrollbars can currently not be overruled in +# Webkit/Chromium, the styling will be left out of the default doxygen.css if +# one or more extra stylesheets have been specified. So if scrollbar +# customization is desired it has to be added explicitly. For an example see the +# documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = @@ -1084,10 +1351,23 @@ HTML_EXTRA_STYLESHEET = HTML_EXTRA_FILES = +# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output +# should be rendered with a dark or light theme. +# Possible values are: LIGHT always generate light mode output, DARK always +# generate dark mode output, AUTO_LIGHT automatically set the mode according to +# the user preference, use light mode if no preference is set (the default), +# AUTO_DARK automatically set the mode according to the user preference, use +# dark mode if no preference is set and TOGGLE allow to user to switch between +# light and dark mode via a button. +# The default value is: AUTO_LIGHT. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE = AUTO_LIGHT + # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see -# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# this color. Hue is specified as an angle on a color-wheel, see +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. @@ -1096,7 +1376,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1114,14 +1394,16 @@ HTML_COLORSTYLE_SAT = 100 HTML_COLORSTYLE_GAMMA = 80 -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via JavaScript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have JavaScript, +# like the Qt help browser. +# The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_TIMESTAMP = NO +HTML_DYNAMIC_MENUS = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the @@ -1146,13 +1428,14 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: http://developer.apple.com/tools/xcode/), introduced with -# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1166,6 +1449,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1191,8 +1481,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1222,7 +1516,7 @@ CHM_FILE = HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). +# (YES) or that it should be included in the main .chm file (NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. @@ -1249,6 +1543,16 @@ BINARY_TOC = NO TOC_EXPAND = NO +# The SITEMAP_URL tag is used to specify the full URL of the place where the +# generated documentation will be placed on the server by the user during the +# deployment of the documentation. The generated sitemap is called sitemap.xml +# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL +# is specified no sitemap is generated. For information about the sitemap +# protocol see https://www.sitemaps.org +# This tag requires that the tag GENERATE_HTML is set to YES. + +SITEMAP_URL = + # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help @@ -1267,7 +1571,8 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1275,8 +1580,8 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- -# folders). +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1284,30 +1589,30 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = @@ -1350,16 +1655,28 @@ DISABLE_INDEX = YES # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1384,6 +1701,24 @@ TREEVIEW_WIDTH = 250 EXT_LINKS_IN_WINDOW = NO +# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email +# addresses. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +OBFUSCATE_EMAILS = YES + +# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see +# https://inkscape.org) to generate formulas as SVG images instead of PNGs for +# the HTML output. These images will generally look nicer at scaled resolutions. +# Possible values are: png (the default) and svg (looks nicer but requires the +# pdf2svg or inkscape tool). +# The default value is: png. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FORMULA_FORMAT = png + # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # doxygen run you need to manually remove any form_*.png images from the HTML @@ -1393,19 +1728,14 @@ EXT_LINKS_IN_WINDOW = NO FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. +# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands +# to create new LaTeX commands to be used in formulas as building blocks. See +# the section "Including formulas" for details. -FORMULA_TRANSPARENT = YES +FORMULA_MACROFILE = # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# http://www.mathjax.org) which uses client side Javascript for the rendering +# https://www.mathjax.org) which uses client side JavaScript for the rendering # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path @@ -1415,11 +1745,29 @@ FORMULA_TRANSPARENT = YES USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1432,22 +1780,29 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from http://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see +# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1475,7 +1830,7 @@ MATHJAX_CODEFILE = SEARCHENGINE = YES # When the SERVER_BASED_SEARCH tag is enabled the search engine will be -# implemented using a web server instead of a web client using Javascript. There +# implemented using a web server instead of a web client using JavaScript. There # are two flavors of web server based searching depending on the EXTERNAL_SEARCH # setting. When disabled, doxygen will generate a PHP script for searching and # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing @@ -1494,7 +1849,8 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). +# Xapian (see: +# https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1507,8 +1863,9 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). See the section "External Indexing and -# Searching" for details. +# Xapian (see: +# https://xapian.org/). See the section "External Indexing and Searching" for +# details. # This tag requires that the tag SEARCHENGINE is set to YES. SEARCHENGINE_URL = @@ -1559,21 +1916,35 @@ LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. # -# Note that when enabling USE_PDFLATEX this option is only used for generating -# bitmaps for formulas in the HTML output, but not in the Makefile that is -# written to the output directory. -# The default file is: latex. +# Note that when not enabling USE_PDFLATEX the default is latex when enabling +# USE_PDFLATEX the default is pdflatex and when in the later case latex is +# chosen this is overwritten by pdflatex. For specific output languages the +# default can have been set differently, this depends on the implementation of +# the output language. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate # index for LaTeX. +# Note: This tag is used in the Makefile / make.bat. +# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file +# (.tex). # The default file is: makeindex. # This tag requires that the tag GENERATE_LATEX is set to YES. MAKEINDEX_CMD_NAME = makeindex +# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to +# generate index for LaTeX. In case there is no backslash (\) as first character +# it will be automatically added in the LaTeX code. +# Note: This tag is used in the generated output file (.tex). +# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat. +# The default value is: makeindex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_MAKEINDEX_CMD = makeindex + # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX # documents. This may be useful for small projects and may help to save some # trees in general. @@ -1603,29 +1974,31 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1658,18 +2031,26 @@ LATEX_EXTRA_FILES = PDF_HYPERLINKS = YES -# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate -# the PDF file directly from the LaTeX files. Set this option to YES, to get a -# higher quality PDF documentation. +# If the USE_PDFLATEX tag is set to YES, doxygen will use the engine as +# specified with LATEX_CMD_NAME to generate the PDF file directly from the LaTeX +# files. Set this option to YES, to get a higher quality PDF documentation. +# +# See also section LATEX_CMD_NAME for selecting the engine. # The default value is: YES. # This tag requires that the tag GENERATE_LATEX is set to YES. USE_PDFLATEX = YES -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode -# command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# The LATEX_BATCHMODE tag ignals the behavior of LaTeX in case of an error. +# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch +# mode nothing is printed on the terminal, errors are scrolled as if is +# hit at every error; missing files that TeX tries to input or request from +# keyboard input (\read on a not open input stream) cause the job to abort, +# NON_STOP In nonstop mode the diagnostic message will appear on the terminal, +# but there is no possibility of user interaction just like in batch mode, +# SCROLL In scroll mode, TeX will stop only for missing files to input or if +# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at +# each error, asking for user intervention. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1684,19 +2065,19 @@ LATEX_HIDE_INDICES = NO # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See -# http://en.wikipedia.org/wiki/BibTeX and \cite for more info. +# https://en.wikipedia.org/wiki/BibTeX and \cite for more info. # The default value is: plain. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_BIB_STYLE = plain -# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated -# page will contain the date and time when the page was generated. Setting this -# to NO can help when comparing the output of multiple runs. -# The default value is: NO. +# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) +# path from which the emoji images will be read. If a relative path is entered, +# it will be relative to the LATEX_OUTPUT directory. If left blank the +# LATEX_OUTPUT directory will be used. # This tag requires that the tag GENERATE_LATEX is set to YES. -LATEX_TIMESTAMP = NO +LATEX_EMOJI_DIRECTORY = #--------------------------------------------------------------------------- # Configuration options related to the RTF output @@ -1737,9 +2118,9 @@ COMPACT_RTF = NO RTF_HYPERLINKS = NO -# Load stylesheet definitions from file. Syntax is similar to doxygen's config -# file, i.e. a series of assignments. You only have to provide replacements, -# missing definitions are set to their default value. +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# configuration file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. # # See also section "Doxygen usage" for information on how to generate the # default style sheet that doxygen normally uses. @@ -1748,8 +2129,8 @@ RTF_HYPERLINKS = NO RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an RTF document. Syntax is -# similar to doxygen's config file. A template extensions file can be generated -# using doxygen -e rtf extensionFile. +# similar to doxygen's configuration file. A template extensions file can be +# generated using doxygen -e rtf extensionFile. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_EXTENSIONS_FILE = @@ -1825,6 +2206,13 @@ XML_OUTPUT = xml XML_PROGRAMLISTING = YES +# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include +# namespace members in file scope as well, matching the HTML output. +# The default value is: NO. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_NS_MEMB_FILE_SCOPE = NO + #--------------------------------------------------------------------------- # Configuration options related to the DOCBOOK output #--------------------------------------------------------------------------- @@ -1848,9 +2236,9 @@ DOCBOOK_OUTPUT = docbook #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sf.net) file that captures the -# structure of the code including all documentation. Note that this feature is -# still experimental and incomplete at the moment. +# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures +# the structure of the code including all documentation. Note that this feature +# is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO @@ -1929,7 +2317,8 @@ SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the -# preprocessor. +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @@ -1950,7 +2339,7 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = NO_COMPAT_IMB_API_053 +PREDEFINED = # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The @@ -2018,16 +2407,9 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- -# Configuration options related to the dot tool +# Configuration options related to diagram generator tools #--------------------------------------------------------------------------- -# You can include diagrams made with dia in doxygen documentation. Doxygen will -# then run dia to produce the diagram and insert it in the documentation. The -# DIA_PATH tag allows you to specify the directory where the dia binary resides. -# If left empty dia is assumed to be found in the default search path. - -DIA_PATH = - # If set to YES the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. @@ -2036,10 +2418,10 @@ HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO -# The default value is: YES. +# The default value is: NO. HAVE_DOT = NO @@ -2053,35 +2435,52 @@ HAVE_DOT = NO DOT_NUM_THREADS = 0 -# When you want a differently looking font in the dot files that doxygen -# generates you can specify the font name using DOT_FONTNAME. You need to make -# sure dot is able to find the font, which can be done by putting it in a -# standard location or by setting the DOTFONTPATH environment variable or by -# setting DOT_FONTPATH to the directory containing the font. -# The default value is: Helvetica. +# DOT_COMMON_ATTR is common attributes for nodes, edges and labels of +# subgraphs. When you want a differently looking font in the dot files that +# doxygen generates you can specify fontname, fontcolor and fontsize attributes. +# For details please see Node, +# Edge and Graph Attributes specification You need to make sure dot is able +# to find the font, which can be done by putting it in a standard location or by +# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. Default graphviz fontsize is 14. +# The default value is: fontname=Helvetica,fontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Helvetica +DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" -# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of -# dot graphs. -# Minimum value: 4, maximum value: 24, default value: 10. +# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can +# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about +# arrows shapes. +# The default value is: labelfontname=Helvetica,labelfontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 10 +DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" -# By default doxygen will tell dot to use the default font as specified with -# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set -# the path where dot can find it using this tag. +# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes +# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification +# The default value is: shape=box,height=0.2,width=0.4. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" + +# You can set the path where dot can find font specified with fontname in +# DOT_COMMON_ATTR and others dot attributes. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then doxygen will +# generate a graph for each documented class showing the direct and indirect +# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and +# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case +# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the +# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used. +# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance +# relations will be shown as texts / links. +# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES @@ -2095,7 +2494,8 @@ CLASS_GRAPH = YES COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. +# groups, showing the direct groups dependencies. See also the chapter Grouping +# in the manual. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2118,10 +2518,32 @@ UML_LOOK = NO # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag HAVE_DOT is set to YES. +# This tag requires that the tag UML_LOOK is set to YES. UML_LIMIT_NUM_FIELDS = 10 +# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and +# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS +# tag is set to YES, doxygen will add type and arguments for attributes and +# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen +# will not generate fields with class member information in the UML graphs. The +# class diagrams will look similar to the default class diagrams but using UML +# notation for the relationships. +# Possible values are: NO, YES and NONE. +# The default value is: NO. +# This tag requires that the tag UML_LOOK is set to YES. + +DOT_UML_DETAILS = NO + +# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters +# to display on a single line. If the actual line length exceeds this threshold +# significantly it will wrapped across multiple lines. Some heuristics are apply +# to avoid ugly line breaks. +# Minimum value: 0, maximum value: 1000, default value: 17. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_WRAP_THRESHOLD = 17 + # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. @@ -2188,16 +2610,21 @@ GRAPHICAL_HIERARCHY = YES DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). +# https://www.graphviz.org/)). # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). -# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd, -# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo, -# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo, +# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, # png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and # png:gdiplus:gdiplus. # The default value is: png. @@ -2230,11 +2657,12 @@ DOT_PATH = DOTFILE_DIRS = -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. -MSCFILE_DIRS = +DIA_PATH = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile @@ -2243,13 +2671,18 @@ MSCFILE_DIRS = DIAFILE_DIRS = # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file. If left blank, it is assumed -# PlantUML is not used or called during a preprocessing step. Doxygen will -# generate a warning when it encounters a \startuml command in this case and -# will not generate output for the diagram. +# path where java can find the plantuml.jar file or to the filename of jar file +# to be used. If left blank, it is assumed PlantUML is not used or called during +# a preprocessing step. Doxygen will generate a warning when it encounters a +# \startuml command in this case and will not generate output for the diagram. PLANTUML_JAR_PATH = +# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a +# configuration file for plantuml. + +PLANTUML_CFG_FILE = + # When using plantuml, the specified paths are searched for files specified by # the !include statement in a plantuml block. @@ -2279,18 +2712,6 @@ DOT_GRAPH_MAX_NODES = 50 MAX_DOT_GRAPH_DEPTH = 0 -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not seem -# to support this out of the box. -# -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_TRANSPARENT = NO - # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) support @@ -2303,14 +2724,34 @@ DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. +# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal +# graphical representation for inheritance and collaboration diagrams is used. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GENERATE_LEGEND = YES -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. +# +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. If the MSCGEN_TOOL tag is left empty (the default), then doxygen will +# use a built-in version of mscgen tool to produce the charts. Alternatively, +# the MSCGEN_TOOL tag can also specify the name an external tool. For instance, +# specifying prog as the value, doxygen will call the tool as prog -T +# -o . The external tool should support +# output file formats "png", "eps", "svg", and "ismap". + +MSCGEN_TOOL = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c deleted file mode 100644 index 484d88bad11580895ae252146db57887f0f7bfb0..0000000000000000000000000000000000000000 --- a/lib/avx/mb_mgr_avx.c +++ /dev/null @@ -1,1024 +0,0 @@ -/******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -#include -#include -#include - -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_avx - -#include "ipsec-mb.h" -#include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" -#include "include/zuc_internal.h" -#include "include/snow3g.h" -#include "include/gcm.h" -#include "include/chacha20_poly1305.h" -#include "include/save_xmms.h" -#include "include/asm.h" -#include "include/des.h" -#include "include/cpu_feature.h" -#include "include/noaesni.h" -#include "include/aesni_emu.h" -#include "include/error.h" - -IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state); - -IMB_JOB *flush_job_zuc256_eea3_avx(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_avx(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -uint32_t hec_32_avx(const uint8_t *in); -uint64_t hec_64_avx(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_chacha20_enc_dec_avx(IMB_JOB *job); - -void *poly1305_mac_scalar(IMB_JOB *job); - -IMB_JOB *snow_v_avx(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); - -#define SAVE_XMMS save_xmms_avx -#define RESTORE_XMMS restore_xmms_avx - -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx - -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_no_gfni_sse -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_no_gfni_sse - -#define AES_CBC_DEC_128 aes_cbc_dec_128_avx -#define AES_CBC_DEC_192 aes_cbc_dec_192_avx -#define AES_CBC_DEC_256 aes_cbc_dec_256_avx - -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx - -#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx -#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx - -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx - -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx - -#define AES_GCM_DEC_128 aes_gcm_dec_128_avx_gen2 -#define AES_GCM_ENC_128 aes_gcm_enc_128_avx_gen2 -#define AES_GCM_DEC_192 aes_gcm_dec_192_avx_gen2 -#define AES_GCM_ENC_192 aes_gcm_enc_192_avx_gen2 -#define AES_GCM_DEC_256 aes_gcm_dec_256_avx_gen2 -#define AES_GCM_ENC_256 aes_gcm_enc_256_avx_gen2 - -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen2 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen2 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen2 - -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx - -#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx -#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx - -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define QUEUE_SIZE queue_size_avx - -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX -#define FLUSH_JOB_AES_DEC FLUSH_JOB_AES_DEC_AVX - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx - -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx -#define POLY1305_MAC poly1305_mac_scalar - -#define SUBMIT_JOB_SNOW_V snow_v_avx -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx - -IMB_JOB *submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -void aes_cmac_256_subkey_gen_avx(const void *key_exp, - void *key1, void *key2); - -#define SUBMIT_JOB_HMAC submit_job_hmac_avx -#define FLUSH_JOB_HMAC flush_job_hmac_avx -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx -#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx -#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx -#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx -#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx -#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx -#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx - -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_avx -#define FLUSH_JOB flush_job_avx -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx -#define GET_NEXT_JOB get_next_job_avx -#define GET_COMPLETED_JOB get_completed_job_avx - -/* ====================================================================== */ - - -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX - -/* ====================================================================== */ - -#define AES_CFB_128_ONE aes_cfb_128_one_avx -#define AES_CFB_256_ONE aes_cfb_256_one_avx - -void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); - -#define AES128_CBC_MAC aes128_cbc_mac_x8 - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx - -/* ====================================================================== */ - -uint32_t -ethernet_fcs_avx_local(const void *msg, const uint64_t len, - const void *tag_ouput); - -#define ETHERNET_FCS ethernet_fcs_avx_local - -uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); -uint32_t crc16_x25_avx(const void *msg, const uint64_t len); -uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len); -uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len); -uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len); -uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); -uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); -uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx - -/* ====================================================================== */ - -/* - * GCM submit / flush API for AVX arch - */ -static IMB_JOB * -submit_job_aes_gcm_dec_avx(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_avx(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -/* ====================================================================== */ - -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_avx(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_bit_avx(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_avx(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_avx(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_avx(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -static void -reset_ooo_mgrs(IMB_MGR *state) -{ - unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - - /* Init AES out-of-order fields */ - memset(aes128_ooo->lens, 0xFF, - sizeof(aes128_ooo->lens)); - memset(&aes128_ooo->lens[0], 0, - sizeof(aes128_ooo->lens[0]) * 8); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xF76543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0xFF, - sizeof(aes192_ooo->lens)); - memset(&aes192_ooo->lens[0], 0, - sizeof(aes192_ooo->lens[0]) * 8); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xF76543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(&aes256_ooo->lens, 0xFF, - sizeof(aes256_ooo->lens)); - memset(&aes256_ooo->lens[0], 0, - sizeof(aes256_ooo->lens[0]) * 8); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xF76543210; - aes256_ooo->num_lanes_inuse = 0; - - /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) - * uses same settings as AES CBC. - */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(&docsis128_sec_ooo->lens[0], 0, - sizeof(docsis128_sec_ooo->lens[0]) * 8); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xF76543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(&docsis128_crc32_sec_ooo->lens[0], 0, - sizeof(docsis128_crc32_sec_ooo->lens[0]) * 8); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(&docsis256_sec_ooo->lens[0], 0, - sizeof(docsis256_sec_ooo->lens[0]) * 8); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xF76543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(&docsis256_crc32_sec_ooo->lens[0], 0, - sizeof(docsis256_crc32_sec_ooo->lens[0]) * 8); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; - - /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFF03020100; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFF03020100; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFF03020100; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; - - /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0xFFFF; - hmac_sha_1_ooo->lens[5] = 0xFFFF; - hmac_sha_1_ooo->lens[6] = 0xFFFF; - hmac_sha_1_ooo->lens[7] = 0xFFFF; - hmac_sha_1_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5*4] = 0x80; - p[64-2] = 0x02; - p[64-1] = 0xA0; - } - /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0xFFFF; - hmac_sha_224_ooo->lens[5] = 0xFFFF; - hmac_sha_224_ooo->lens[6] = 0xFFFF; - hmac_sha_224_ooo->lens[7] = 0xFFFF; - hmac_sha_224_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7 * 4] = 0x80; /* digest 7 words long */ - p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ - p[64 - 1] = 0xE0; - } - - /* Init HMAC/SHA256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0xFFFF; - hmac_sha_256_ooo->lens[5] = 0xFFFF; - hmac_sha_256_ooo->lens[6] = 0xFFFF; - hmac_sha_256_ooo->lens[7] = 0xFFFF; - hmac_sha_256_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); - p[8 * 4] = 0x80; /* 8 digest words */ - p[64 - 2] = 0x03; /* length */ - p[64 - 1] = 0x00; - } - - - /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0xFFFF; - hmac_sha_384_ooo->lens[3] = 0xFFFF; - hmac_sha_384_ooo->lens[4] = 0xFFFF; - hmac_sha_384_ooo->lens[5] = 0xFFFF; - hmac_sha_384_ooo->lens[6] = 0xFFFF; - hmac_sha_384_ooo->lens[7] = 0xFFFF; - hmac_sha_384_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < AVX_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 384 bits, 1408 bits == 0x0580. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } - - /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0xFFFF; - hmac_sha_512_ooo->lens[3] = 0xFFFF; - hmac_sha_512_ooo->lens[4] = 0xFFFF; - hmac_sha_512_ooo->lens[5] = 0xFFFF; - hmac_sha_512_ooo->lens[6] = 0xFFFF; - hmac_sha_512_ooo->lens[7] = 0xFFFF; - hmac_sha_512_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < AVX_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; - /* - * hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 512 bits, 1536 bits == 0x600. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } - - - /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0xFFFF; - hmac_md5_ooo->lens[9] = 0xFFFF; - hmac_md5_ooo->lens[10] = 0xFFFF; - hmac_md5_ooo->lens[11] = 0xFFFF; - hmac_md5_ooo->lens[12] = 0xFFFF; - hmac_md5_ooo->lens[13] = 0xFFFF; - hmac_md5_ooo->lens[14] = 0xFFFF; - hmac_md5_ooo->lens[15] = 0xFFFF; - hmac_md5_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4 * 4] = 0x80; - p[64 - 7] = 0x02; - p[64 - 8] = 0x80; - } - - /* Init AES/XCBC OOO fields */ - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < 8; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].job_in_lane = NULL; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15); - } - aes_xcbc_ooo->num_lanes_inuse = 0; - - /* Init AES-CCM auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_ccm_ooo->init_done[j] = 0; - aes_ccm_ooo->lens[j] = 0; - aes_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes_ccm_ooo->lens[j] = 0xFFFF; - - aes_ccm_ooo->unused_lanes = 0xF76543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_ccm_ooo->init_done[j] = 0; - aes256_ccm_ooo->lens[j] = 0; - aes256_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes256_ccm_ooo->lens[j] = 0xFFFF; - - aes256_ccm_ooo->unused_lanes = 0xF76543210; - aes256_ccm_ooo->num_lanes_inuse = 0; - - /* Init AES-CMAC auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_cmac_ooo->init_done[j] = 0; - aes_cmac_ooo->lens[j] = 0; - aes_cmac_ooo->job_in_lane[j] = NULL; - } - aes_cmac_ooo->unused_lanes = 0xF76543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_cmac_ooo->init_done[j] = 0; - aes256_cmac_ooo->lens[j] = 0; - aes256_cmac_ooo->job_in_lane[j] = NULL; - } - aes256_cmac_ooo->unused_lanes = 0xF76543210; - aes256_cmac_ooo->num_lanes_inuse = 0; - - /* Init AES CBC-S out-of-order fields */ - memset(aes128_cbcs_ooo->lens, 0xFF, - sizeof(aes128_cbcs_ooo->lens)); - memset(&aes128_cbcs_ooo->lens[0], 0, - sizeof(aes128_cbcs_ooo->lens[0]) * 8); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xF76543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; -} - -IMB_DLL_LOCAL void -init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) -{ -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_AVX; - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set AVX handlers */ - state->get_next_job = get_next_job_avx; - state->submit_job = submit_job_avx; - state->submit_job_nocheck = submit_job_nocheck_avx; - state->get_completed_job = get_completed_job_avx; - state->flush_job = flush_job_avx; - state->queue_size = queue_size_avx; - state->keyexp_128 = aes_keyexp_128_avx; - state->keyexp_192 = aes_keyexp_192_avx; - state->keyexp_256 = aes_keyexp_256_avx; - state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx; - state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx; - state->xcbc_keyexp = aes_xcbc_expand_key_avx; - state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx; - state->sha1 = sha1_avx; - state->sha224_one_block = sha224_one_block_avx; - state->sha224 = sha224_avx; - state->sha256_one_block = sha256_one_block_avx; - state->sha256 = sha256_avx; - state->sha384_one_block = sha384_one_block_avx; - state->sha384 = sha384_avx; - state->sha512_one_block = sha512_one_block_avx; - state->sha512 = sha512_avx; - state->md5_one_block = md5_one_block_avx; - state->aes128_cfb_one = aes_cfb_128_one_avx; - - state->eea3_1_buffer = zuc_eea3_1_buffer_avx; - state->eea3_4_buffer = zuc_eea3_4_buffer_avx; - state->eea3_n_buffer = zuc_eea3_n_buffer_avx; - state->eia3_1_buffer = zuc_eia3_1_buffer_avx; - state->eia3_n_buffer = zuc_eia3_n_buffer_avx; - - state->f8_1_buffer = kasumi_f8_1_buffer_avx; - state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; - state->f8_2_buffer = kasumi_f8_2_buffer_avx; - state->f8_3_buffer = kasumi_f8_3_buffer_avx; - state->f8_4_buffer = kasumi_f8_4_buffer_avx; - state->f8_n_buffer = kasumi_f8_n_buffer_avx; - state->f9_1_buffer = kasumi_f9_1_buffer_avx; - state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; - state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; - state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; - state->kasumi_key_sched_size = kasumi_key_sched_size_avx; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx; - state->snow3g_init_key_sched = snow3g_init_key_sched_avx; - state->snow3g_key_sched_size = snow3g_key_sched_size_avx; - - state->gcm128_enc = aes_gcm_enc_128_avx_gen2; - state->gcm192_enc = aes_gcm_enc_192_avx_gen2; - state->gcm256_enc = aes_gcm_enc_256_avx_gen2; - state->gcm128_dec = aes_gcm_dec_128_avx_gen2; - state->gcm192_dec = aes_gcm_dec_192_avx_gen2; - state->gcm256_dec = aes_gcm_dec_256_avx_gen2; - state->gcm128_init = aes_gcm_init_128_avx_gen2; - state->gcm192_init = aes_gcm_init_192_avx_gen2; - state->gcm256_init = aes_gcm_init_256_avx_gen2; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen2; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen2; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen2; - state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen2; - state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen2; - state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen2; - state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen2; - state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen2; - state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen2; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen2; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen2; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen2; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen2; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen2; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen2; - state->gcm128_precomp = aes_gcm_precomp_128_avx_gen2; - state->gcm192_precomp = aes_gcm_precomp_192_avx_gen2; - state->gcm256_precomp = aes_gcm_precomp_256_avx_gen2; - state->gcm128_pre = aes_gcm_pre_128_avx_gen2; - state->gcm192_pre = aes_gcm_pre_192_avx_gen2; - state->gcm256_pre = aes_gcm_pre_256_avx_gen2; - state->ghash = ghash_avx_gen2; - state->ghash_pre = ghash_pre_avx_gen2; - - state->gmac128_init = imb_aes_gmac_init_128_avx_gen2; - state->gmac192_init = imb_aes_gmac_init_192_avx_gen2; - state->gmac256_init = imb_aes_gmac_init_256_avx_gen2; - state->gmac128_update = imb_aes_gmac_update_128_avx_gen2; - state->gmac192_update = imb_aes_gmac_update_192_avx_gen2; - state->gmac256_update = imb_aes_gmac_update_256_avx_gen2; - state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen2; - state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen2; - state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen2; - state->hec_32 = hec_32_avx; - state->hec_64 = hec_64_avx; - state->crc32_ethernet_fcs = ethernet_fcs_avx; - state->crc16_x25 = crc16_x25_avx; - state->crc32_sctp = crc32_sctp_avx; - state->crc24_lte_a = crc24_lte_a_avx; - state->crc24_lte_b = crc24_lte_b_avx; - state->crc16_fp_data = crc16_fp_data_avx; - state->crc11_fp_header = crc11_fp_header_avx; - state->crc7_fp_header = crc7_fp_header_avx; - state->crc10_iuup_data = crc10_iuup_data_avx; - state->crc6_iuup_header = crc6_iuup_header_avx; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; - - state->chacha20_poly1305_init = init_chacha20_poly1305_avx; - state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx; - state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx; - state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; -} - -void -init_mb_mgr_avx(IMB_MGR *state) -{ - init_mb_mgr_avx_internal(state, 1); -} - -#include "mb_mgr_code.h" diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c deleted file mode 100644 index 70b7dc4c1777c817c3580ffc966926e2be675e35..0000000000000000000000000000000000000000 --- a/lib/avx2/mb_mgr_avx2.c +++ /dev/null @@ -1,965 +0,0 @@ -/******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -#include -#include -#include - -#define AVX2 -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_ymms - -#include "ipsec-mb.h" -#include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" -#include "include/zuc_internal.h" -#include "include/snow3g.h" -#include "include/gcm.h" -#include "include/chacha20_poly1305.h" - -#include "include/save_xmms.h" -#include "include/asm.h" -#include "include/des.h" -#include "include/cpu_feature.h" -#include "include/noaesni.h" -#include "include/aesni_emu.h" -#include "include/error.h" - -IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state); - -IMB_JOB *flush_job_zuc256_eea3_avx2(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_avx2(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -void aes_cmac_256_subkey_gen_avx2(const void *key_exp, - void *key1, void *key2); -uint32_t hec_32_avx(const uint8_t *in); -uint64_t hec_64_avx(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_chacha20_enc_dec_avx2(IMB_JOB *job); - -void *poly1305_mac_scalar(IMB_JOB *job); - -IMB_JOB *snow_v_avx(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); - -#define SAVE_XMMS save_xmms_avx -#define RESTORE_XMMS restore_xmms_avx - -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx - -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx - -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx - -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx - -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx2 -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx2 -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx2 -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx2 -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx2 -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx2 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_no_gfni_sse -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_no_gfni_sse - -#define AES_CBC_DEC_128 aes_cbc_dec_128_avx -#define AES_CBC_DEC_192 aes_cbc_dec_192_avx -#define AES_CBC_DEC_256 aes_cbc_dec_256_avx - -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx - -#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx -#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx - -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx - -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx - -#define AES_GCM_DEC_128 aes_gcm_dec_128_avx_gen4 -#define AES_GCM_ENC_128 aes_gcm_enc_128_avx_gen4 -#define AES_GCM_DEC_192 aes_gcm_dec_192_avx_gen4 -#define AES_GCM_ENC_192 aes_gcm_enc_192_avx_gen4 -#define AES_GCM_DEC_256 aes_gcm_dec_256_avx_gen4 -#define AES_GCM_ENC_256 aes_gcm_enc_256_avx_gen4 - -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen4 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen4 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen4 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen4 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen4 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen4 - -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx2 -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx2 - -#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx -#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx - -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define QUEUE_SIZE queue_size_avx2 - -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX2 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX2 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX2 - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx2 -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx2 -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx2 -#define POLY1305_MAC poly1305_mac_scalar - -#define SUBMIT_JOB_SNOW_V snow_v_avx -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx - -IMB_JOB *submit_job_hmac_avx2(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_avx2(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_avx2(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_avx2(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_avx2(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_avx2(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_avx2(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_avx2(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_avx2(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_avx2(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - - -IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -#define SUBMIT_JOB_HMAC submit_job_hmac_avx2 -#define FLUSH_JOB_HMAC flush_job_hmac_avx2 -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx2 -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx2 -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx2 -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx2 -#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx2 -#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx2 -#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx2 -#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx2 -#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 -#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 - -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_avx2 -#define FLUSH_JOB flush_job_avx2 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx2 -#define QUEUE_SIZE queue_size_avx2 -#define GET_NEXT_JOB get_next_job_avx2 -#define GET_COMPLETED_JOB get_completed_job_avx2 - -/* ====================================================================== */ - -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX2 -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX2 - -/* ====================================================================== */ - -#define AES_CFB_128_ONE aes_cfb_128_one_avx2 -#define AES_CFB_256_ONE aes_cfb_256_one_avx2 - -void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); - -#define AES128_CBC_MAC aes128_cbc_mac_x8 - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx - -/* ====================================================================== */ - -uint32_t ethernet_fcs_avx_local(const void *msg, const uint64_t len, - const void *tag_ouput); - -#define ETHERNET_FCS ethernet_fcs_avx_local - -uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); -uint32_t crc16_x25_avx(const void *msg, const uint64_t len); -uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len); -uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len); -uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len); -uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); -uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); -uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx - -/* ====================================================================== */ - -/* - * GCM submit / flush API for AVX2 arch - */ -static IMB_JOB * -submit_job_aes_gcm_dec_avx2(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_avx2(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -/* ====================================================================== */ - -static void -reset_ooo_mgrs(IMB_MGR *state) -{ - unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - - /* Init AES out-of-order fields */ - memset(aes128_ooo->lens, 0xFF, - sizeof(aes128_ooo->lens)); - memset(&aes128_ooo->lens[0], 0, - sizeof(aes128_ooo->lens[0]) * 8); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xF76543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0xFF, - sizeof(aes192_ooo->lens)); - memset(&aes192_ooo->lens[0], 0, - sizeof(aes192_ooo->lens[0]) * 8); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xF76543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(&aes256_ooo->lens, 0xFF, - sizeof(aes256_ooo->lens)); - memset(&aes256_ooo->lens[0], 0, - sizeof(aes256_ooo->lens[0]) * 8); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xF76543210; - aes256_ooo->num_lanes_inuse = 0; - - /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) - * uses same settings as AES CBC. - */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(&docsis128_sec_ooo->lens[0], 0, - sizeof(docsis128_sec_ooo->lens[0]) * 8); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xF76543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(&docsis128_crc32_sec_ooo->lens[0], 0, - sizeof(docsis128_crc32_sec_ooo->lens[0]) * 8); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(&docsis256_sec_ooo->lens[0], 0, - sizeof(docsis256_sec_ooo->lens[0]) * 8); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xF76543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(&docsis256_crc32_sec_ooo->lens[0], 0, - sizeof(docsis256_crc32_sec_ooo->lens[0]) * 8); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; - - /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xF76543210; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0xff; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xF76543210; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0xff; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xF76543210; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0xff; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; - - /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0; - hmac_sha_1_ooo->lens[5] = 0; - hmac_sha_1_ooo->lens[6] = 0; - hmac_sha_1_ooo->lens[7] = 0; - hmac_sha_1_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX2_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5 * 4] = 0x80; - p[64 - 2] = 0x02; - p[64 - 1] = 0xA0; - } - /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0; - hmac_sha_224_ooo->lens[5] = 0; - hmac_sha_224_ooo->lens[6] = 0; - hmac_sha_224_ooo->lens[7] = 0; - hmac_sha_224_ooo->unused_lanes = 0xF76543210; - /* sha256 and sha224 are very similar except for - * digest constants and output size - */ - for (j = 0; j < AVX2_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7 * 4] = 0x80; /* digest 7 words long */ - p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ - p[64 - 1] = 0xE0; - } - - /* Init HMAC/SHA256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0; - hmac_sha_256_ooo->lens[5] = 0; - hmac_sha_256_ooo->lens[6] = 0; - hmac_sha_256_ooo->lens[7] = 0; - hmac_sha_256_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX2_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - /* hmac related */ - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); - p[8 * 4] = 0x80; /* 8 digest words */ - p[64 - 2] = 0x03; /* length */ - p[64 - 1] = 0x00; - } - - /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0; - hmac_sha_384_ooo->lens[3] = 0; - hmac_sha_384_ooo->lens[4] = 0xFFFF; - hmac_sha_384_ooo->lens[5] = 0xFFFF; - hmac_sha_384_ooo->lens[6] = 0xFFFF; - hmac_sha_384_ooo->lens[7] = 0xFFFF; - hmac_sha_384_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX2_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 384 bits, 1408 bits == 0x0580. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } - - /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0; - hmac_sha_512_ooo->lens[3] = 0; - hmac_sha_512_ooo->lens[4] = 0xFFFF; - hmac_sha_512_ooo->lens[5] = 0xFFFF; - hmac_sha_512_ooo->lens[6] = 0xFFFF; - hmac_sha_512_ooo->lens[7] = 0xFFFF; - hmac_sha_512_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX2_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 512 bits, 1536 bits == 0x600. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } - - /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0; - hmac_md5_ooo->lens[9] = 0; - hmac_md5_ooo->lens[10] = 0; - hmac_md5_ooo->lens[11] = 0; - hmac_md5_ooo->lens[12] = 0; - hmac_md5_ooo->lens[13] = 0; - hmac_md5_ooo->lens[14] = 0; - hmac_md5_ooo->lens[15] = 0; - hmac_md5_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_md5_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX2_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4 * 4] = 0x80; - p[64 - 7] = 0x02; - p[64 - 8] = 0x80; - } - - /* Init AES/XCBC OOO fields */ - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < 8; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15); - } - aes_xcbc_ooo->num_lanes_inuse = 0; - - /* Init AES-CCM auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_ccm_ooo->init_done[j] = 0; - aes_ccm_ooo->lens[j] = 0; - aes_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes_ccm_ooo->lens[j] = 0xFFFF; - - aes_ccm_ooo->unused_lanes = 0xF76543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_ccm_ooo->init_done[j] = 0; - aes256_ccm_ooo->lens[j] = 0; - aes256_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes256_ccm_ooo->lens[j] = 0xFFFF; - - aes256_ccm_ooo->unused_lanes = 0xF76543210; - aes256_ccm_ooo->num_lanes_inuse = 0; - - /* Init AES-CMAC auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_cmac_ooo->init_done[j] = 0; - aes_cmac_ooo->lens[j] = 0; - aes_cmac_ooo->job_in_lane[j] = NULL; - } - aes_cmac_ooo->unused_lanes = 0xF76543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_cmac_ooo->init_done[j] = 0; - aes256_cmac_ooo->lens[j] = 0; - aes256_cmac_ooo->job_in_lane[j] = NULL; - } - aes256_cmac_ooo->unused_lanes = 0xF76543210; - aes256_cmac_ooo->num_lanes_inuse = 0; - - /* Init AES CBC-S out-of-order fields */ - memset(aes128_cbcs_ooo->lens, 0xFF, - sizeof(aes128_cbcs_ooo->lens)); - memset(&aes128_cbcs_ooo->lens[0], 0, - sizeof(aes128_cbcs_ooo->lens[0]) * 8); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xF76543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; -} - -IMB_DLL_LOCAL void -init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) -{ -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_AVX2; - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set handlers */ - state->get_next_job = get_next_job_avx2; - state->submit_job = submit_job_avx2; - state->submit_job_nocheck = submit_job_nocheck_avx2; - state->get_completed_job = get_completed_job_avx2; - state->flush_job = flush_job_avx2; - state->queue_size = queue_size_avx2; - state->keyexp_128 = aes_keyexp_128_avx2; - state->keyexp_192 = aes_keyexp_192_avx2; - state->keyexp_256 = aes_keyexp_256_avx2; - state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx2; - state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx2; - state->xcbc_keyexp = aes_xcbc_expand_key_avx2; - state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx2; - state->sha1 = sha1_avx2; - state->sha224_one_block = sha224_one_block_avx2; - state->sha224 = sha224_avx2; - state->sha256_one_block = sha256_one_block_avx2; - state->sha256 = sha256_avx2; - state->sha384_one_block = sha384_one_block_avx2; - state->sha384 = sha384_avx2; - state->sha512_one_block = sha512_one_block_avx2; - state->sha512 = sha512_avx2; - state->md5_one_block = md5_one_block_avx2; - state->aes128_cfb_one = aes_cfb_128_one_avx2; - - state->eea3_1_buffer = zuc_eea3_1_buffer_avx2; - state->eea3_4_buffer = zuc_eea3_4_buffer_avx; - state->eea3_n_buffer = zuc_eea3_n_buffer_avx2; - state->eia3_1_buffer = zuc_eia3_1_buffer_avx2; - state->eia3_n_buffer = zuc_eia3_n_buffer_avx2; - - state->f8_1_buffer = kasumi_f8_1_buffer_avx; - state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; - state->f8_2_buffer = kasumi_f8_2_buffer_avx; - state->f8_3_buffer = kasumi_f8_3_buffer_avx; - state->f8_4_buffer = kasumi_f8_4_buffer_avx; - state->f8_n_buffer = kasumi_f8_n_buffer_avx; - state->f9_1_buffer = kasumi_f9_1_buffer_avx; - state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; - state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; - state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; - state->kasumi_key_sched_size = kasumi_key_sched_size_avx; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx2; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx2; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx2; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx2; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx2; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx2; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx2; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx2; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx2; - state->snow3g_init_key_sched = snow3g_init_key_sched_avx2; - state->snow3g_key_sched_size = snow3g_key_sched_size_avx2; - - state->gcm128_enc = aes_gcm_enc_128_avx_gen4; - state->gcm192_enc = aes_gcm_enc_192_avx_gen4; - state->gcm256_enc = aes_gcm_enc_256_avx_gen4; - state->gcm128_dec = aes_gcm_dec_128_avx_gen4; - state->gcm192_dec = aes_gcm_dec_192_avx_gen4; - state->gcm256_dec = aes_gcm_dec_256_avx_gen4; - state->gcm128_init = aes_gcm_init_128_avx_gen4; - state->gcm192_init = aes_gcm_init_192_avx_gen4; - state->gcm256_init = aes_gcm_init_256_avx_gen4; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen4; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen4; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen4; - state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen4; - state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen4; - state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen4; - state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen4; - state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen4; - state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen4; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen4; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen4; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen4; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen4; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen4; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen4; - state->gcm128_precomp = aes_gcm_precomp_128_avx_gen4; - state->gcm192_precomp = aes_gcm_precomp_192_avx_gen4; - state->gcm256_precomp = aes_gcm_precomp_256_avx_gen4; - state->gcm128_pre = aes_gcm_pre_128_avx_gen4; - state->gcm192_pre = aes_gcm_pre_192_avx_gen4; - state->gcm256_pre = aes_gcm_pre_256_avx_gen4; - state->ghash = ghash_avx_gen4; - state->ghash_pre = ghash_pre_avx_gen2; - - state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; - state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; - state->gmac256_init = imb_aes_gmac_init_256_avx_gen4; - state->gmac128_update = imb_aes_gmac_update_128_avx_gen4; - state->gmac192_update = imb_aes_gmac_update_192_avx_gen4; - state->gmac256_update = imb_aes_gmac_update_256_avx_gen4; - state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; - state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; - state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; - state->hec_32 = hec_32_avx; - state->hec_64 = hec_64_avx; - state->crc32_ethernet_fcs = ethernet_fcs_avx; - state->crc16_x25 = crc16_x25_avx; - state->crc32_sctp = crc32_sctp_avx; - state->crc24_lte_a = crc24_lte_a_avx; - state->crc24_lte_b = crc24_lte_b_avx; - state->crc16_fp_data = crc16_fp_data_avx; - state->crc11_fp_header = crc11_fp_header_avx; - state->crc7_fp_header = crc7_fp_header_avx; - state->crc10_iuup_data = crc10_iuup_data_avx; - state->crc6_iuup_header = crc6_iuup_header_avx; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; - - state->chacha20_poly1305_init = init_chacha20_poly1305_avx; - state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx2; - state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx2; - state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; -} - -void -init_mb_mgr_avx2(IMB_MGR *state) -{ - init_mb_mgr_avx2_internal(state, 1); -} - -#include "mb_mgr_code.h" diff --git a/lib/avx2_t1/aes128_gcm_by8_avx2.asm b/lib/avx2_t1/aes128_gcm_by8_avx2.asm index 789453183c5ef0f4313a46baf2f71128e4f7b7b5..be3a4d15722c92885d1bdba60ee7e48e52d3a318 100644 --- a/lib/avx2_t1/aes128_gcm_by8_avx2.asm +++ b/lib/avx2_t1/aes128_gcm_by8_avx2.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2017-2022, Intel Corporation All rights reserved. +; Copyright(c) 2017-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx2_t1/gcm_avx_gen4.asm" +%include "include/gcm_avx_gen4.inc" diff --git a/lib/avx2_t1/aes192_gcm_by8_avx2.asm b/lib/avx2_t1/aes192_gcm_by8_avx2.asm index 6f3255a303f89b3c4f5a22d99c80defdab74c844..4d28c0d6795b1e1932f2ff52079ece680da0c8b8 100644 --- a/lib/avx2_t1/aes192_gcm_by8_avx2.asm +++ b/lib/avx2_t1/aes192_gcm_by8_avx2.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2017-2022, Intel Corporation All rights reserved. +; Copyright(c) 2017-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx2_t1/gcm_avx_gen4.asm" +%include "include/gcm_avx_gen4.inc" diff --git a/lib/avx2_t1/aes256_gcm_by8_avx2.asm b/lib/avx2_t1/aes256_gcm_by8_avx2.asm index 4e3bf23e64025931be13f8f2f30b30ca017d1b98..63c87273160091e2b148d6a328b9d2a193bd6cb5 100644 --- a/lib/avx2_t1/aes256_gcm_by8_avx2.asm +++ b/lib/avx2_t1/aes256_gcm_by8_avx2.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2017-2022, Intel Corporation All rights reserved. +; Copyright(c) 2017-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx2_t1/gcm_avx_gen4.asm" +%include "include/gcm_avx_gen4.inc" diff --git a/lib/avx2_t1/chacha20_avx2.asm b/lib/avx2_t1/chacha20_avx2.asm index 25e116fd17c7d289a7d9301189941ff51c809a79..7c65f2fb0df1a8c90d5e15927cfb65a805dc9f0e 100644 --- a/lib/avx2_t1/chacha20_avx2.asm +++ b/lib/avx2_t1/chacha20_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" -%include "include/transpose_avx2.asm" -%include "include/chacha_poly_defines.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" +%include "include/transpose_avx2.inc" +%include "include/chacha_poly_defines.inc" %include "include/cet.inc" mksection .rodata default rel diff --git a/lib/avx2_t1/gcm_avx_gen4.asm b/lib/avx2_t1/gcm_avx_gen4.asm deleted file mode 100644 index 189280bd9fb5261925bea21db9e3c32da2e9582d..0000000000000000000000000000000000000000 --- a/lib/avx2_t1/gcm_avx_gen4.asm +++ /dev/null @@ -1,4552 +0,0 @@ -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2011-2022, Intel Corporation All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions -; are met: -; * Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; * Redistributions in binary form must reproduce the above copyright -; notice, this list of conditions and the following disclaimer in -; the documentation and/or other materials provided with the -; distribution. -; * Neither the name of Intel Corporation nor the names of its -; contributors may be used to endorse or promote products derived -; from this software without specific prior written permission. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -; -; Authors: -; Erdinc Ozturk -; Vinodh Gopal -; James Guilford -; -; -; References: -; This code was derived and highly optimized from the code described in paper: -; Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on Intel Architecture Processors. August, 2010 -; The details of the implementation is explained in: -; Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode on Intel Architecture Processors. October, 2012. -; -; -; -; -; Assumptions: -; -; -; -; iv: -; 0 1 2 3 -; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | Salt (From the SA) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | Initialization Vector | -; | (This is the sequence number from IPSec header) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 0x1 | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; -; -; -; AAD: -; AAD will be padded with 0 to the next 16byte multiple -; for example, assume AAD is a u32 vector -; -; if AAD is 8 bytes: -; AAD[3] = {A0, A1}; -; padded AAD in xmm register = {A1 A0 0 0} -; -; 0 1 2 3 -; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | SPI (A1) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 32-bit Sequence Number (A0) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 0x0 | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; -; AAD Format with 32-bit Sequence Number -; -; if AAD is 12 bytes: -; AAD[3] = {A0, A1, A2}; -; padded AAD in xmm register = {A2 A1 A0 0} -; -; 0 1 2 3 -; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | SPI (A2) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 64-bit Extended Sequence Number {A1,A0} | -; | | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 0x0 | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; -; AAD Format with 64-bit Extended Sequence Number -; -; -; aadLen: -; Must be a multiple of 4 bytes and from the definition of the spec. -; The code additionally supports any aadLen length. -; -; TLen: -; from the definition of the spec, TLen can only be 8, 12 or 16 bytes. -; -; poly = x^128 + x^127 + x^126 + x^121 + 1 -; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part. -; - -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" -%include "include/gcm_defines.asm" -%include "include/gcm_keys_avx2_avx512.asm" -%include "include/gcm_common.inc" -%include "include/memcpy.asm" -%include "include/cet.inc" -%include "include/error.inc" -%ifndef GCM128_MODE -%ifndef GCM192_MODE -%ifndef GCM256_MODE -%error "No GCM mode selected for gcm_avx_gen4.asm!" -%endif -%endif -%endif - -;; Decide on AES-GCM key size to compile for -%ifdef GCM128_MODE -%define NROUNDS 9 -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ avx_gen4 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _128_ %+ avx_gen4 -%endif - -%ifdef GCM192_MODE -%define NROUNDS 11 -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ avx_gen4 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _192_ %+ avx_gen4 -%endif - -%ifdef GCM256_MODE -%define NROUNDS 13 -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ avx_gen4 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _256_ %+ avx_gen4 -%endif - -mksection .text -default rel - -; need to push 4 registers into stack to maintain -%define STACK_OFFSET 8*4 - -%define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register) -%define TMP3 16*1 ; Temporary storage for AES State 3 -%define TMP4 16*2 ; Temporary storage for AES State 4 -%define TMP5 16*3 ; Temporary storage for AES State 5 -%define TMP6 16*4 ; Temporary storage for AES State 6 -%define TMP7 16*5 ; Temporary storage for AES State 7 -%define TMP8 16*6 ; Temporary storage for AES State 8 - -%define LOCAL_STORAGE 16*7 - -%ifidn __OUTPUT_FORMAT__, win64 - %define XMM_STORAGE 16*10 -%else - %define XMM_STORAGE 0 -%endif - -%define VARIABLE_OFFSET LOCAL_STORAGE + XMM_STORAGE - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Utility Macros -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) -; Input: A and B (128-bits each, bit-reflected) -; Output: C = A*B*x mod poly, (i.e. >>1 ) -; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input -; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GHASH_MUL 7 -%define %%GH %1 ; 16 Bytes -%define %%HK %2 ; 16 Bytes -%define %%T1 %3 -%define %%T2 %4 -%define %%T3 %5 -%define %%T4 %6 -%define %%T5 %7 - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vpclmulqdq %%T1, %%GH, %%HK, 0x11 ; %%T1 = a1*b1 - vpclmulqdq %%T2, %%GH, %%HK, 0x00 ; %%T2 = a0*b0 - vpclmulqdq %%T3, %%GH, %%HK, 0x01 ; %%T3 = a1*b0 - vpclmulqdq %%GH, %%GH, %%HK, 0x10 ; %%GH = a0*b1 - vpxor %%GH, %%GH, %%T3 - - vpsrldq %%T3, %%GH, 8 ; shift-R %%GH 2 DWs - vpslldq %%GH, %%GH, 8 ; shift-L %%GH 2 DWs - - vpxor %%T1, %%T1, %%T3 - vpxor %%GH, %%GH, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%T3, [rel POLY2] - - vpclmulqdq %%T2, %%T3, %%GH, 0x01 - vpslldq %%T2, %%T2, 8 ; shift-L %%T2 2 DWs - - vpxor %%GH, %%GH, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;second phase of the reduction - vpclmulqdq %%T2, %%T3, %%GH, 0x00 - vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%GH, %%T3, %%GH, 0x10 - vpslldq %%GH, %%GH, 4 ; shift-L %%GH 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%GH, %%GH, %%T2 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%GH, %%GH, %%T1 ; the result is in %%GH - -%endmacro - -; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx_gen4 -; functions, but are kept to allow users to switch cpu architectures between calls -; of pre, init, update, and finalize. -%macro PRECOMPUTE 8 -%define %%GDATA %1 -%define %%HK %2 -%define %%T1 %3 -%define %%T2 %4 -%define %%T3 %5 -%define %%T4 %6 -%define %%T5 %7 -%define %%T6 %8 - - ; Haskey_i_k holds XORed values of the low and high parts of the Haskey_i - vmovdqa %%T5, %%HK - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^2<<1 mod poly - vmovdqu [%%GDATA + HashKey_2], %%T5 ; [HashKey_2] = HashKey^2<<1 mod poly - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^3<<1 mod poly - vmovdqu [%%GDATA + HashKey_3], %%T5 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^4<<1 mod poly - vmovdqu [%%GDATA + HashKey_4], %%T5 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^5<<1 mod poly - vmovdqu [%%GDATA + HashKey_5], %%T5 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^6<<1 mod poly - vmovdqu [%%GDATA + HashKey_6], %%T5 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^7<<1 mod poly - vmovdqu [%%GDATA + HashKey_7], %%T5 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^8<<1 mod poly - vmovdqu [%%GDATA + HashKey_8], %%T5 -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. -; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). -; Output: The hash of the data (AAD_HASH). -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro CALC_AAD_HASH 15 -%define %%A_IN %1 -%define %%A_LEN %2 -%define %%AAD_HASH %3 -%define %%GDATA_KEY %4 -%define %%XTMP0 %5 ; xmm temp reg 5 -%define %%XTMP1 %6 ; xmm temp reg 5 -%define %%XTMP2 %7 -%define %%XTMP3 %8 -%define %%XTMP4 %9 -%define %%XTMP5 %10 ; xmm temp reg 5 -%define %%T1 %11 ; temp reg 1 -%define %%T2 %12 -%define %%T3 %13 -%define %%T4 %14 -%define %%T5 %15 ; temp reg 5 - - mov %%T1, %%A_IN ; T1 = AAD - mov %%T2, %%A_LEN ; T2 = aadLen - -%%_get_AAD_loop128: - cmp %%T2, 128 - jl %%_exit_AAD_loop128 - - vmovdqu %%XTMP0, [%%T1 + 16*0] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vpxor %%XTMP0, %%AAD_HASH - - vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_8] - vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 - vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 - vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 - -%assign i 1 -%assign j 7 -%rep 7 - vmovdqu %%XTMP0, [%%T1 + 16*i] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_ %+ j] - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 - vpxor %%XTMP1, %%XTMP1, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 -%assign i (i + 1) -%assign j (j - 1) -%endrep - - vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs - vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%XTMP5, [rel POLY2] - vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 - vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;second phase of the reduction - vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 - vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 - vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 - - sub %%T2, 128 - je %%_CALC_AAD_done - - add %%T1, 128 - jmp %%_get_AAD_loop128 - -%%_exit_AAD_loop128: - cmp %%T2, 16 - jl %%_get_small_AAD_block - - ;; calculate hash_key position to start with - mov %%T3, %%T2 - and %%T3, -16 ; 1 to 7 blocks possible here - neg %%T3 - add %%T3, HashKey_1 + 16 - lea %%T3, [%%GDATA_KEY + %%T3] - - vmovdqu %%XTMP0, [%%T1] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vpxor %%XTMP0, %%AAD_HASH - - vmovdqu %%XTMP5, [%%T3] - vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 - vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 - vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 - - add %%T3, 16 ; move to next hashkey - add %%T1, 16 ; move to next data block - sub %%T2, 16 - cmp %%T2, 16 - jl %%_AAD_reduce - -%%_AAD_blocks: - vmovdqu %%XTMP0, [%%T1] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vmovdqu %%XTMP5, [%%T3] - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 - vpxor %%XTMP1, %%XTMP1, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 - - add %%T3, 16 ; move to next hashkey - add %%T1, 16 - sub %%T2, 16 - cmp %%T2, 16 - jl %%_AAD_reduce - jmp %%_AAD_blocks - -%%_AAD_reduce: - vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs - vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%XTMP5, [rel POLY2] - vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 - vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;second phase of the reduction - vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 - vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 - vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 - -%%_get_small_AAD_block: - or %%T2, %%T2 - je %%_CALC_AAD_done - - vmovdqu %%XTMP0, [%%GDATA_KEY + HashKey] - READ_SMALL_DATA_INPUT_AVX %%XTMP1, %%T1, %%T2, %%T3, %%T4, %%T5 - ;byte-reflect the AAD data - vpshufb %%XTMP1, [rel SHUF_MASK] - vpxor %%AAD_HASH, %%XTMP1 - GHASH_MUL %%AAD_HASH, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 - -%%_CALC_AAD_done: - -%endmacro ; CALC_AAD_HASH - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks between update calls. -; Requires the input data be at least 1 byte long. -; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), -; input text length (PLAIN_CYPH_LEN), the current data offset (DATA_OFFSET), -; the hash subkey (HASH_SUBKEY) and whether encoding or decoding (ENC_DEC) -; Output: A cypher of the first partial block (CYPH_PLAIN_OUT), and updated GDATA_CTX -; Clobbers rax, r10, r12, r13, r15, xmm0, xmm1, xmm2, xmm3, xmm5, xmm6, xmm9, xmm10, xmm11 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro PARTIAL_BLOCK 8 -%define %%GDATA_CTX %1 -%define %%CYPH_PLAIN_OUT %2 -%define %%PLAIN_CYPH_IN %3 -%define %%PLAIN_CYPH_LEN %4 -%define %%DATA_OFFSET %5 -%define %%AAD_HASH %6 -%define %%HASH_SUBKEY %7 -%define %%ENC_DEC %8 - - mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 - je %%_partial_block_done ;Leave Macro if no partial blocks - - cmp %%PLAIN_CYPH_LEN, 16 ;Read in input data without over reading - jl %%_fewer_than_16_bytes - VXLDR xmm1, [%%PLAIN_CYPH_IN] ;If more than 16 bytes of data, just fill the xmm register - jmp %%_data_read - -%%_fewer_than_16_bytes: - lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - READ_SMALL_DATA_INPUT_AVX xmm1, r10, %%PLAIN_CYPH_LEN, rax, r12, r15 - -%%_data_read: ;Finished reading in data - - vmovdqu xmm9, [%%GDATA_CTX + PBlockEncKey] ;xmm9 = my_ctx_data.partial_block_enc_key - - lea r12, [rel SHIFT_MASK] - - add r12, r13 ; adjust the shuffle mask pointer to be able to shift r13 bytes (16-r13 is the number of bytes in plaintext mod 16) - vmovdqu xmm2, [r12] ; get the appropriate shuffle mask - vpshufb xmm9, xmm2 ;shift right r13 bytes - -%ifidn %%ENC_DEC, DEC - vmovdqa xmm3, xmm1 - vpxor xmm9, xmm1 ; Ciphertext XOR E(K, Yn) - - mov r15, %%PLAIN_CYPH_LEN - add r15, r13 - sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block - jge %%_no_extra_mask_1 ;Determine if if partial block is not being filled and shift mask accordingly - sub r12, r15 -%%_no_extra_mask_1: - - vmovdqu xmm1, [r12 + ALL_F - SHIFT_MASK]; get the appropriate mask to mask out bottom r13 bytes of xmm9 - vpand xmm9, xmm1 ; mask out bottom r13 bytes of xmm9 - - vpand xmm3, xmm1 - vpshufb xmm3, [rel SHUF_MASK] - vpshufb xmm3, xmm2 - vpxor %%AAD_HASH, xmm3 - - cmp r15,0 - jl %%_partial_incomplete_1 - - GHASH_MUL %%AAD_HASH, %%HASH_SUBKEY, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block - xor rax,rax - mov [%%GDATA_CTX + PBlockLen], rax - jmp %%_dec_done -%%_partial_incomplete_1: -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_CYPH_LEN - add [%%GDATA_CTX + PBlockLen], rax -%else - add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN -%endif -%%_dec_done: - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH - -%else - vpxor xmm9, xmm1 ; Plaintext XOR E(K, Yn) - - mov r15, %%PLAIN_CYPH_LEN - add r15, r13 - sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block - jge %%_no_extra_mask_2 ;Determine if if partial block is not being filled and shift mask accordingly - sub r12, r15 -%%_no_extra_mask_2: - - vmovdqu xmm1, [r12 + ALL_F-SHIFT_MASK] ; get the appropriate mask to mask out bottom r13 bytes of xmm9 - vpand xmm9, xmm1 ; mask out bottom r13 bytes of xmm9 - - vpshufb xmm9, [rel SHUF_MASK] - vpshufb xmm9, xmm2 - vpxor %%AAD_HASH, xmm9 - - cmp r15,0 - jl %%_partial_incomplete_2 - - GHASH_MUL %%AAD_HASH, %%HASH_SUBKEY, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block - xor rax,rax - mov [%%GDATA_CTX + PBlockLen], rax - jmp %%_encode_done -%%_partial_incomplete_2: -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_CYPH_LEN - add [%%GDATA_CTX + PBlockLen], rax -%else - add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN -%endif -%%_encode_done: - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH - - vpshufb xmm9, [rel SHUF_MASK] ; shuffle xmm9 back to output as ciphertext - vpshufb xmm9, xmm2 -%endif - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ; output encrypted Bytes - cmp r15,0 - jl %%_partial_fill - mov r12, r13 - mov r13, 16 - sub r13, r12 ; Set r13 to be the number of bytes to write out - jmp %%_count_set -%%_partial_fill: - mov r13, %%PLAIN_CYPH_LEN -%%_count_set: - vmovq rax, xmm9 - cmp r13, 8 - jle %%_less_than_8_bytes_left - - mov [%%CYPH_PLAIN_OUT+ %%DATA_OFFSET], rax - add %%DATA_OFFSET, 8 - vpsrldq xmm9, xmm9, 8 - vmovq rax, xmm9 - sub r13, 8 -%%_less_than_8_bytes_left: - mov BYTE [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], al - add %%DATA_OFFSET, 1 - shr rax, 8 - sub r13, 1 - jne %%_less_than_8_bytes_left - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%%_partial_block_done: -%endmacro ; PARTIAL_BLOCK - -%macro GHASH_SINGLE_MUL 9 -%define %%GDATA %1 -%define %%HASHKEY %2 -%define %%CIPHER %3 -%define %%STATE_11 %4 -%define %%STATE_00 %5 -%define %%STATE_MID %6 -%define %%T1 %7 -%define %%T2 %8 -%define %%FIRST %9 - - vmovdqu %%T1, [%%GDATA + %%HASHKEY] -%ifidn %%FIRST, first - vpclmulqdq %%STATE_11, %%CIPHER, %%T1, 0x11 ; %%T4 = a1*b1 - vpclmulqdq %%STATE_00, %%CIPHER, %%T1, 0x00 ; %%T4_2 = a0*b0 - vpclmulqdq %%STATE_MID, %%CIPHER, %%T1, 0x01 ; %%T6 = a1*b0 - vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 ; %%T5 = a0*b1 - vpxor %%STATE_MID, %%STATE_MID, %%T2 -%else - vpclmulqdq %%T2, %%CIPHER, %%T1, 0x11 - vpxor %%STATE_11, %%STATE_11, %%T2 - - vpclmulqdq %%T2, %%CIPHER, %%T1, 0x00 - vpxor %%STATE_00, %%STATE_00, %%T2 - - vpclmulqdq %%T2, %%CIPHER, %%T1, 0x01 - vpxor %%STATE_MID, %%STATE_MID, %%T2 - - vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 - vpxor %%STATE_MID, %%STATE_MID, %%T2 -%endif - -%endmacro - -; if a = number of total plaintext bytes -; b = floor(a/16) -; %%num_initial_blocks = b mod 8; -; encrypt the initial %%num_initial_blocks blocks and apply ghash on the ciphertext -; %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r14 are used as a pointer only, not modified. -; Updated AAD_HASH is returned in %%T3 - -%macro INITIAL_BLOCKS 23 -%define %%GDATA_KEY %1 -%define %%CYPH_PLAIN_OUT %2 -%define %%PLAIN_CYPH_IN %3 -%define %%LENGTH %4 -%define %%DATA_OFFSET %5 -%define %%num_initial_blocks %6 ; can be 0, 1, 2, 3, 4, 5, 6 or 7 -%define %%T1 %7 -%define %%T2 %8 -%define %%T3 %9 -%define %%T4 %10 -%define %%T5 %11 -%define %%CTR %12 -%define %%XMM1 %13 -%define %%XMM2 %14 -%define %%XMM3 %15 -%define %%XMM4 %16 -%define %%XMM5 %17 -%define %%XMM6 %18 -%define %%XMM7 %19 -%define %%XMM8 %20 -%define %%T6 %21 -%define %%T_key %22 -%define %%ENC_DEC %23 - -%assign i (8-%%num_initial_blocks) - ;; Move AAD_HASH to temp reg - vmovdqu %%T2, %%XMM8 - ;; Start AES for %%num_initial_blocks blocks - ;; vmovdqu %%CTR, [%%GDATA_CTX + CurCount] ; %%CTR = Y0 - -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 - vmovdqa reg(i), %%CTR - vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap -%assign i (i+1) -%endrep - -%if(%%num_initial_blocks>0) -vmovdqu %%T_key, [%%GDATA_KEY+16*0] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vpxor reg(i),reg(i),%%T_key -%assign i (i+1) -%endrep - -%assign j 1 -%rep NROUNDS -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vaesenc reg(i),%%T_key -%assign i (i+1) -%endrep - -%assign j (j+1) -%endrep - -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vaesenclast reg(i),%%T_key -%assign i (i+1) -%endrep - -%endif ; %if(%%num_initial_blocks>0) - -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - vpxor reg(i), reg(i), %%T1 - ;; Write back ciphertext for %%num_initial_blocks blocks - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) - add %%DATA_OFFSET, 16 - %ifidn %%ENC_DEC, DEC - vmovdqa reg(i), %%T1 - %endif - ;; Prepare ciphertext for GHASH computations - vpshufb reg(i), [rel SHUF_MASK] -%assign i (i+1) -%endrep - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%assign i (9-%%num_initial_blocks) -%if(%%num_initial_blocks>0) - vmovdqa %%T3, reg(i) -%assign i (i+1) -%endif -%if(%%num_initial_blocks>1) -%rep %%num_initial_blocks-1 - vmovdqu [rsp + TMP %+ i], reg(i) -%assign i (i+1) -%endrep -%endif - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;; Prepare 8 counter blocks and perform rounds of AES cipher on - ;; them, load plain/cipher text and store cipher/plain text. - ;; Stitch GHASH computation in between AES rounds. - vpaddd %%XMM1, %%CTR, [rel ONE] ; INCR Y0 - vpaddd %%XMM2, %%CTR, [rel TWO] ; INCR Y0 - vpaddd %%XMM3, %%XMM1, [rel TWO] ; INCR Y0 - vpaddd %%XMM4, %%XMM2, [rel TWO] ; INCR Y0 - vpaddd %%XMM5, %%XMM3, [rel TWO] ; INCR Y0 - vpaddd %%XMM6, %%XMM4, [rel TWO] ; INCR Y0 - vpaddd %%XMM7, %%XMM5, [rel TWO] ; INCR Y0 - vpaddd %%XMM8, %%XMM6, [rel TWO] ; INCR Y0 - vmovdqa %%CTR, %%XMM8 - - vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap - - vmovdqu %%T_key, [%%GDATA_KEY+16*0] - vpxor %%XMM1, %%XMM1, %%T_key - vpxor %%XMM2, %%XMM2, %%T_key - vpxor %%XMM3, %%XMM3, %%T_key - vpxor %%XMM4, %%XMM4, %%T_key - vpxor %%XMM5, %%XMM5, %%T_key - vpxor %%XMM6, %%XMM6, %%T_key - vpxor %%XMM7, %%XMM7, %%T_key - vpxor %%XMM8, %%XMM8, %%T_key - -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) -%assign k (%%num_initial_blocks) - -%define %%T4_2 %%T4 -%if(%%num_initial_blocks>0) - ;; Hash in AES state - ;; T2 - incoming AAD hash - vpxor %%T2, %%T3 - - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, first -%endif - - vmovdqu %%T_key, [%%GDATA_KEY+16*1] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - - vmovdqu %%T_key, [%%GDATA_KEY+16*2] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%if(%%num_initial_blocks>1) - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - vmovdqu %%T2, [rsp + TMP %+ j] - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, not_first -%endif - - vmovdqu %%T_key, [%%GDATA_KEY+16*3] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - - vmovdqu %%T_key, [%%GDATA_KEY+16*4] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%if(%%num_initial_blocks>2) - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - vmovdqu %%T2, [rsp + TMP %+ j] - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, not_first -%endif - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%if(%%num_initial_blocks>3) - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - vmovdqu %%T2, [rsp + TMP %+ j] - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, not_first -%endif - - vmovdqu %%T_key, [%%GDATA_KEY+16*5] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - - vmovdqu %%T_key, [%%GDATA_KEY+16*6] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%if(%%num_initial_blocks>4) - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - vmovdqu %%T2, [rsp + TMP %+ j] - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, not_first -%endif - - vmovdqu %%T_key, [%%GDATA_KEY+16*7] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - - vmovdqu %%T_key, [%%GDATA_KEY+16*8] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%if(%%num_initial_blocks>5) - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - vmovdqu %%T2, [rsp + TMP %+ j] - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, not_first -%endif - - vmovdqu %%T_key, [%%GDATA_KEY+16*9] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - -%ifndef GCM128_MODE - vmovdqu %%T_key, [%%GDATA_KEY+16*10] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key -%endif - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%if(%%num_initial_blocks>6) - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - vmovdqu %%T2, [rsp + TMP %+ j] - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, not_first -%endif - -%ifdef GCM128_MODE - vmovdqu %%T_key, [%%GDATA_KEY+16*10] - vaesenclast %%XMM1, %%T_key - vaesenclast %%XMM2, %%T_key - vaesenclast %%XMM3, %%T_key - vaesenclast %%XMM4, %%T_key - vaesenclast %%XMM5, %%T_key - vaesenclast %%XMM6, %%T_key - vaesenclast %%XMM7, %%T_key - vaesenclast %%XMM8, %%T_key -%endif - -%ifdef GCM192_MODE - vmovdqu %%T_key, [%%GDATA_KEY+16*11] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - - vmovdqu %%T_key, [%%GDATA_KEY+16*12] - vaesenclast %%XMM1, %%T_key - vaesenclast %%XMM2, %%T_key - vaesenclast %%XMM3, %%T_key - vaesenclast %%XMM4, %%T_key - vaesenclast %%XMM5, %%T_key - vaesenclast %%XMM6, %%T_key - vaesenclast %%XMM7, %%T_key - vaesenclast %%XMM8, %%T_key -%endif -%ifdef GCM256_MODE - vmovdqu %%T_key, [%%GDATA_KEY+16*11] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - - vmovdqu %%T_key, [%%GDATA_KEY+16*12] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key -%endif - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%if(%%num_initial_blocks>7) - ;; GDATA, HASHKEY, CIPHER, - ;; STATE_11, STATE_00, STATE_MID, T1, T2 - vmovdqu %%T2, [rsp + TMP %+ j] - GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ - %%T1, %%T4, %%T6, %%T5, %%T3, not_first -%endif - -%ifdef GCM256_MODE ; GCM256 - vmovdqu %%T_key, [%%GDATA_KEY+16*13] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key - - vmovdqu %%T_key, [%%GDATA_KEY+16*14] - vaesenclast %%XMM1, %%T_key - vaesenclast %%XMM2, %%T_key - vaesenclast %%XMM3, %%T_key - vaesenclast %%XMM4, %%T_key - vaesenclast %%XMM5, %%T_key - vaesenclast %%XMM6, %%T_key - vaesenclast %%XMM7, %%T_key - vaesenclast %%XMM8, %%T_key -%endif ; GCM256 mode - -%if(%%num_initial_blocks>0) - vpsrldq %%T3, %%T6, 8 ; shift-R %%T2 2 DWs - vpslldq %%T6, %%T6, 8 ; shift-L %%T3 2 DWs - vpxor %%T1, %%T1, %%T3 ; accumulate the results in %%T1:%%T4 - vpxor %%T4, %%T6, %%T4 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ; First phase of the reduction - vmovdqa %%T3, [rel POLY2] - - vpclmulqdq %%T2, %%T3, %%T4, 0x01 - vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs - - ;; First phase of the reduction complete - vpxor %%T4, %%T4, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ; Second phase of the reduction - vpclmulqdq %%T2, %%T3, %%T4, 0x00 - ;; Shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - vpsrldq %%T2, %%T2, 4 - - vpclmulqdq %%T4, %%T3, %%T4, 0x10 - ;; Shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) - vpslldq %%T4, %%T4, 4 - ;; Second phase of the reduction complete - vpxor %%T4, %%T4, %%T2 - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ; The result is in %%T3 - vpxor %%T3, %%T1, %%T4 -%else - ;; The hash should end up in T3 - vmovdqa %%T3, %%T2 -%endif - - ;; Final hash is now in T3 -%if %%num_initial_blocks > 0 - ;; NOTE: obsolete in case %%num_initial_blocks = 0 - sub %%LENGTH, 16*%%num_initial_blocks -%endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*0] - vpxor %%XMM1, %%XMM1, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*0], %%XMM1 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM1, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*1] - vpxor %%XMM2, %%XMM2, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*1], %%XMM2 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM2, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*2] - vpxor %%XMM3, %%XMM3, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*2], %%XMM3 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM3, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*3] - vpxor %%XMM4, %%XMM4, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*3], %%XMM4 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM4, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*4] - vpxor %%XMM5, %%XMM5, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*4], %%XMM5 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM5, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*5] - vpxor %%XMM6, %%XMM6, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*5], %%XMM6 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM6, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*6] - vpxor %%XMM7, %%XMM7, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*6], %%XMM7 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM7, %%T1 - %endif - -%if %%num_initial_blocks > 0 - ;; NOTE: 'jl' is never taken for %%num_initial_blocks = 0 - ;; This macro is executed for length 128 and up, - ;; zero length is checked in GCM_ENC_DEC. - ;; If the last block is partial then the xor will be done later - ;; in ENCRYPT_FINAL_PARTIAL_BLOCK. - ;; We know it's partial if LENGTH - 16*num_initial_blocks < 128 - cmp %%LENGTH, 128 - jl %%_initial_skip_last_word_write -%endif - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*7] - vpxor %%XMM8, %%XMM8, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*7], %%XMM8 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM8, %%T1 - %endif - - ;; Update %%LENGTH with the number of blocks processed - sub %%LENGTH, 16 - add %%DATA_OFFSET, 16 -%%_initial_skip_last_word_write: - sub %%LENGTH, 128-16 - add %%DATA_OFFSET, 128-16 - - vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap - ;; Combine GHASHed value with the corresponding ciphertext - vpxor %%XMM1, %%XMM1, %%T3 - vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%%_initial_blocks_done: - -%endmacro - -;;; INITIAL_BLOCKS macro with support for a partial final block. -;;; num_initial_blocks is expected to include the partial final block -;;; in the count. -%macro INITIAL_BLOCKS_PARTIAL 25 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%CYPH_PLAIN_OUT %3 -%define %%PLAIN_CYPH_IN %4 -%define %%LENGTH %5 -%define %%DATA_OFFSET %6 -%define %%num_initial_blocks %7 ; can be 1, 2, 3, 4, 5, 6 or 7 (not 0) -%define %%T1 %8 -%define %%T2 %9 -%define %%T3 %10 -%define %%T4 %11 -%define %%T5 %12 -%define %%CTR %13 -%define %%XMM1 %14 -%define %%XMM2 %15 -%define %%XMM3 %16 -%define %%XMM4 %17 -%define %%XMM5 %18 -%define %%XMM6 %19 -%define %%XMM7 %20 -%define %%XMM8 %21 -%define %%T6 %22 -%define %%T_key %23 -%define %%ENC_DEC %24 -%define %%INSTANCE_TYPE %25 - -%assign i (8-%%num_initial_blocks) - ;; Move AAD_HASH to temp reg - vmovdqu %%T2, %%XMM8 - ;; vmovdqu %%CTR, [%%GDATA_CTX + CurCount] ; %%CTR = Y0 - -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - ;; Compute AES counters - vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 - vmovdqa reg(i), %%CTR - vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap -%assign i (i+1) -%endrep - -vmovdqu %%T_key, [%%GDATA_KEY+16*0] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - ; Start AES for %%num_initial_blocks blocks - vpxor reg(i),reg(i),%%T_key -%assign i (i+1) -%endrep - -%assign j 1 -%rep NROUNDS -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vaesenc reg(i),%%T_key -%assign i (i+1) -%endrep - -%assign j (j+1) -%endrep - -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vaesenclast reg(i),%%T_key -%assign i (i+1) -%endrep - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; Hash all but the last block of data -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks-1 - ;; Encrypt the message for all but the last block - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - vpxor reg(i), reg(i), %%T1 - ;; write back ciphertext for %%num_initial_blocks blocks - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) - add %%DATA_OFFSET, 16 - %ifidn %%ENC_DEC, DEC - vmovdqa reg(i), %%T1 - %endif - ;; Prepare ciphertext for GHASH computations - vpshufb reg(i), [rel SHUF_MASK] -%assign i (i+1) -%endrep - - ;; The final block of data may be <16B - sub %%LENGTH, 16*(%%num_initial_blocks-1) - -%if %%num_initial_blocks < 8 - ;; NOTE: the 'jl' is always taken for num_initial_blocks = 8. - ;; This is run in the context of GCM_ENC_DEC_SMALL for length < 128. - cmp %%LENGTH, 16 - jl %%_small_initial_partial_block - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; Handle a full length final block - encrypt and hash all blocks -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - sub %%LENGTH, 16 - mov [%%GDATA_CTX + PBlockLen], %%LENGTH - - ;; Encrypt the message - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - vpxor reg(i), reg(i), %%T1 - ;; write back ciphertext for %%num_initial_blocks blocks - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) - add %%DATA_OFFSET, 16 - %ifidn %%ENC_DEC, DEC - vmovdqa reg(i), %%T1 - %endif - ;; Prepare ciphertext for GHASH computations - vpshufb reg(i), [rel SHUF_MASK] - - ;; Hash all of the data -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) -%assign k (%%num_initial_blocks) -%assign last_block_to_hash 0 - -%if(%%num_initial_blocks>last_block_to_hash) - ;; Hash in AES state - vpxor %%T2, reg(j) - - ;; T2 - incoming AAD hash - ;; reg(i) holds ciphertext - ;; T5 - hash key - ;; T6 - updated xor - ;; reg(1)/xmm1 should now be available for tmp use - vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] - vpclmulqdq %%T1, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 - vpclmulqdq %%T4, %%T2, %%T5, 0x00 ; %%T4 = a0*b0 - vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 - vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 - vpxor %%T6, %%T6, %%T5 -%endif - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%assign rep_count (%%num_initial_blocks-1) -%rep rep_count - - vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] - vpclmulqdq %%T3, reg(j), %%T5, 0x11 - vpxor %%T1, %%T1, %%T3 - - vpclmulqdq %%T3, reg(j), %%T5, 0x00 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, reg(j), %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, reg(j), %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%endrep - - ;; Record that a reduction is needed - mov r12, 1 - - jmp %%_small_initial_compute_hash - -%endif ; %if %%num_initial_blocks < 8 - -%%_small_initial_partial_block: - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; Handle ghash for a <16B final block -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - ;; In this case if it's a single call to encrypt we can - ;; hash all of the data but if it's an init / update / finalize - ;; series of call we need to leave the last block if it's - ;; less than a full block of data. - - mov [%%GDATA_CTX + PBlockLen], %%LENGTH - vmovdqu [%%GDATA_CTX + PBlockEncKey], reg(i) - ;; Handle a partial final block - ;; GDATA, KEY, T1, T2 - ;; r13 - length - ;; LT16 - indicates type of read and that the buffer is less than 16 bytes long - ;; NOTE: could be replaced with %%LENGTH but at this point - ;; %%LENGTH is always less than 16. - ;; No PLAIN_CYPH_LEN argument available in this macro. - ENCRYPT_FINAL_PARTIAL_BLOCK reg(i), %%T1, %%T3, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, LT16, %%ENC_DEC, %%DATA_OFFSET - vpshufb reg(i), [rel SHUF_MASK] - -%ifidn %%INSTANCE_TYPE, multi_call -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) -%assign k (%%num_initial_blocks-1) -%assign last_block_to_hash 1 -%else -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) -%assign k (%%num_initial_blocks) -%assign last_block_to_hash 0 -%endif - -%if(%%num_initial_blocks>last_block_to_hash) - ;; Record that a reduction is needed - mov r12, 1 - ;; Hash in AES state - vpxor %%T2, reg(j) - - ;; T2 - incoming AAD hash - ;; reg(i) holds ciphertext - ;; T5 - hash key - ;; T6 - updated xor - ;; reg(1)/xmm1 should now be available for tmp use - vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] - vpclmulqdq %%T1, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 - vpclmulqdq %%T4, %%T2, %%T5, 0x00 ; %%T4 = a0*b0 - vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 - vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 - vpxor %%T6, %%T6, %%T5 -%else - ;; Record that a reduction is not needed - - ;; In this case no hashes are computed because there - ;; is only one initial block and it is < 16B in length. - mov r12, 0 -%endif - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%ifidn %%INSTANCE_TYPE, multi_call -%assign rep_count (%%num_initial_blocks-2) -%%_multi_call_hash: -%else -%assign rep_count (%%num_initial_blocks-1) -%endif - -%if rep_count < 0 - ;; quick fix for negative rep_count (to be investigated) -%assign rep_count 0 -%endif - -%rep rep_count - - vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] - vpclmulqdq %%T3, reg(j), %%T5, 0x11 - vpxor %%T1, %%T1, %%T3 - - vpclmulqdq %%T3, reg(j), %%T5, 0x00 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, reg(j), %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, reg(j), %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - -%assign i (i+1) -%assign j (j+1) -%assign k (k-1) -%endrep - -%%_small_initial_compute_hash: - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; Ghash reduction -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%if(%%num_initial_blocks=1) -%ifidn %%INSTANCE_TYPE, multi_call - ;; We only need to check if a reduction is needed if - ;; initial_blocks == 1 and init/update/final is being used. - ;; In this case we may just have a partial block, and that - ;; gets hashed in finalize. - cmp r12, 0 - je %%_no_reduction_needed -%endif -%endif - - vpsrldq %%T3, %%T6, 8 ; shift-R %%T2 2 DWs - vpslldq %%T6, %%T6, 8 ; shift-L %%T3 2 DWs - vpxor %%T1, %%T1, %%T3 ; accumulate the results in %%T1:%%T4 - vpxor %%T4, %%T6, %%T4 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;; First phase of the reduction - vmovdqa %%T3, [rel POLY2] - - vpclmulqdq %%T2, %%T3, %%T4, 0x01 - ;; shift-L xmm2 2 DWs - vpslldq %%T2, %%T2, 8 - vpxor %%T4, %%T4, %%T2 - - ;; First phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;; Second phase of the reduction - - vpclmulqdq %%T2, %%T3, %%T4, 0x00 - ;; Shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - vpsrldq %%T2, %%T2, 4 - - vpclmulqdq %%T4, %%T3, %%T4, 0x10 - ;; Shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) - vpslldq %%T4, %%T4, 4 - - vpxor %%T4, %%T4, %%T2 - ;; Second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%T3, %%T1, %%T4 - -%ifidn %%INSTANCE_TYPE, multi_call - ;; If using init/update/finalize, we need to xor any partial block data - ;; into the hash. -%if %%num_initial_blocks > 1 - ;; NOTE: for %%num_initial_blocks = 0 the xor never takes place -%if %%num_initial_blocks != 8 - ;; NOTE: for %%num_initial_blocks = 8, %%LENGTH, stored in [PBlockLen] is never zero - cmp qword [%%GDATA_CTX + PBlockLen], 0 - je %%_no_partial_block_xor -%endif ; %%num_initial_blocks != 8 - vpxor %%T3, %%T3, reg(8) -%%_no_partial_block_xor: -%endif ; %%num_initial_blocks > 1 -%endif ; %%INSTANCE_TYPE, multi_call - -%if(%%num_initial_blocks=1) -%ifidn %%INSTANCE_TYPE, multi_call - ;; NOTE: %%_no_reduction_needed case only valid for - ;; multi_call with initial_blocks = 1. - ;; Look for comment above around '_no_reduction_needed' - ;; The jmp below is obsolete as the code will fall through. - - ;; The result is in %%T3 - jmp %%_after_reduction - -%%_no_reduction_needed: - ;; The hash should end up in T3. The only way we should get here is if - ;; there is a partial block of data, so xor that into the hash. - vpxor %%T3, %%T2, reg(8) -%endif ; %%INSTANCE_TYPE = multi_call -%endif ; %%num_initial_blocks=1 - -%%_after_reduction: - ;; Final hash is now in T3 - -%endmacro ; INITIAL_BLOCKS_PARTIAL - -; encrypt 8 blocks at a time -; ghash the 8 previously encrypted ciphertext blocks -; %%GDATA (KEY), %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN are used as pointers only, not modified -; %%DATA_OFFSET is the data offset value -%macro GHASH_8_ENCRYPT_8_PARALLEL 23 -%define %%GDATA %1 -%define %%CYPH_PLAIN_OUT %2 -%define %%PLAIN_CYPH_IN %3 -%define %%DATA_OFFSET %4 -%define %%T1 %5 -%define %%T2 %6 -%define %%T3 %7 -%define %%T4 %8 -%define %%T5 %9 -%define %%T6 %10 -%define %%CTR %11 -%define %%XMM1 %12 -%define %%XMM2 %13 -%define %%XMM3 %14 -%define %%XMM4 %15 -%define %%XMM5 %16 -%define %%XMM6 %17 -%define %%XMM7 %18 -%define %%XMM8 %19 -%define %%T7 %20 -%define %%loop_idx %21 -%define %%ENC_DEC %22 -%define %%FULL_PARTIAL %23 - - vmovdqa %%T2, %%XMM1 - vmovdqu [rsp + TMP2], %%XMM2 - vmovdqu [rsp + TMP3], %%XMM3 - vmovdqu [rsp + TMP4], %%XMM4 - vmovdqu [rsp + TMP5], %%XMM5 - vmovdqu [rsp + TMP6], %%XMM6 - vmovdqu [rsp + TMP7], %%XMM7 - vmovdqu [rsp + TMP8], %%XMM8 - -%ifidn %%loop_idx, in_order - vpaddd %%XMM1, %%CTR, [rel ONE] ; INCR CNT - vmovdqa %%T5, [rel TWO] - vpaddd %%XMM2, %%CTR, %%T5 - vpaddd %%XMM3, %%XMM1, %%T5 - vpaddd %%XMM4, %%XMM2, %%T5 - vpaddd %%XMM5, %%XMM3, %%T5 - vpaddd %%XMM6, %%XMM4, %%T5 - vpaddd %%XMM7, %%XMM5, %%T5 - vpaddd %%XMM8, %%XMM6, %%T5 - vmovdqa %%CTR, %%XMM8 - - vmovdqa %%T5, [rel SHUF_MASK] - vpshufb %%XMM1, %%T5 ; perform a 16Byte swap - vpshufb %%XMM2, %%T5 ; perform a 16Byte swap - vpshufb %%XMM3, %%T5 ; perform a 16Byte swap - vpshufb %%XMM4, %%T5 ; perform a 16Byte swap - vpshufb %%XMM5, %%T5 ; perform a 16Byte swap - vpshufb %%XMM6, %%T5 ; perform a 16Byte swap - vpshufb %%XMM7, %%T5 ; perform a 16Byte swap - vpshufb %%XMM8, %%T5 ; perform a 16Byte swap -%else - vpaddd %%XMM1, %%CTR, [rel ONEf] ; INCR CNT - vmovdqa %%T5, [rel TWOf] - vpaddd %%XMM2, %%CTR, %%T5 - vpaddd %%XMM3, %%XMM1, %%T5 - vpaddd %%XMM4, %%XMM2, %%T5 - vpaddd %%XMM5, %%XMM3, %%T5 - vpaddd %%XMM6, %%XMM4, %%T5 - vpaddd %%XMM7, %%XMM5, %%T5 - vpaddd %%XMM8, %%XMM6, %%T5 - vmovdqa %%CTR, %%XMM8 -%endif - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T1, [%%GDATA + 16*0] - vpxor %%XMM1, %%XMM1, %%T1 - vpxor %%XMM2, %%XMM2, %%T1 - vpxor %%XMM3, %%XMM3, %%T1 - vpxor %%XMM4, %%XMM4, %%T1 - vpxor %%XMM5, %%XMM5, %%T1 - vpxor %%XMM6, %%XMM6, %%T1 - vpxor %%XMM7, %%XMM7, %%T1 - vpxor %%XMM8, %%XMM8, %%T1 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T1, [%%GDATA + 16*1] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [%%GDATA + 16*2] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_8] - vpclmulqdq %%T4, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 - vpclmulqdq %%T7, %%T2, %%T5, 0x00 ; %%T7 = a0*b0 - vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 - vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 - vpxor %%T6, %%T6, %%T5 - - vmovdqu %%T1, [%%GDATA + 16*3] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP2] - vmovdqu %%T5, [%%GDATA + HashKey_7] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*4] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vmovdqu %%T1, [rsp + TMP3] - vmovdqu %%T5, [%%GDATA + HashKey_6] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*5] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP4] - vmovdqu %%T5, [%%GDATA + HashKey_5] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*6] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP5] - vmovdqu %%T5, [%%GDATA + HashKey_4] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*7] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP6] - vmovdqu %%T5, [%%GDATA + HashKey_3] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*8] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP7] - vmovdqu %%T5, [%%GDATA + HashKey_2] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + 16*9] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T1, [rsp + TMP8] - vmovdqu %%T5, [%%GDATA + HashKey] - - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x01 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T1, %%T4, %%T3 - - vmovdqu %%T5, [%%GDATA + 16*10] - %ifndef GCM128_MODE ; GCM192 or GCM256 - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*11] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*12] -%endif -%ifdef GCM256_MODE - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*13] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*14] -%endif ; GCM256 - -%assign i 0 -%assign j 1 -%rep 8 - - ;; SNP TBD: This is pretty ugly - consider whether just XORing the - ;; data in after vaesenclast is simpler and performant. Would - ;; also have to ripple it through partial block and ghash_mul_8. -%ifidn %%FULL_PARTIAL, full - %ifdef NT_LD - VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - vpxor %%T2, %%T2, %%T5 - %else - vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - %endif - - %ifidn %%ENC_DEC, ENC - vaesenclast reg(j), reg(j), %%T2 - %else - vaesenclast %%T3, reg(j), %%T2 - vpxor reg(j), %%T2, %%T5 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 - %endif - -%else - ; Don't read the final data during partial block processing - %ifdef NT_LD - %if (i<7) - VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - vpxor %%T2, %%T2, %%T5 - %else - ;; Stage the key directly in T2 rather than hash it with plaintext - vmovdqu %%T2, %%T5 - %endif - %else - %if (i<7) - vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - %else - ;; Stage the key directly in T2 rather than hash it with plaintext - vmovdqu %%T2, %%T5 - %endif - %endif - - %ifidn %%ENC_DEC, ENC - vaesenclast reg(j), reg(j), %%T2 - %else - %if (i<7) - vaesenclast %%T3, reg(j), %%T2 - vpxor reg(j), %%T2, %%T5 - ;; Do not read the data since it could fault - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 - %else - vaesenclast reg(j), reg(j), %%T2 - %endif - %endif -%endif - -%assign i (i+1) -%assign j (j+1) -%endrep - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vpslldq %%T3, %%T6, 8 ; shift-L %%T3 2 DWs - vpsrldq %%T6, %%T6, 8 ; shift-R %%T2 2 DWs - vpxor %%T7, %%T7, %%T3 - vpxor %%T1, %%T1, %%T6 ; accumulate the results in %%T1:%%T7 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%T3, [rel POLY2] - - vpclmulqdq %%T2, %%T3, %%T7, 0x01 - vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs - - vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - %ifidn %%ENC_DEC, ENC - ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*0], %%XMM1 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*1], %%XMM2 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*2], %%XMM3 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*3], %%XMM4 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*4], %%XMM5 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*5], %%XMM6 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*6], %%XMM7 - %ifidn %%FULL_PARTIAL, full - ;; Avoid writing past the buffer if handling a partial block - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*7], %%XMM8 - %endif - %endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;second phase of the reduction - vpclmulqdq %%T2, %%T3, %%T7, 0x00 - vpsrldq %%T2, %%T2, 4 ; shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%T4, %%T3, %%T7, 0x10 - vpslldq %%T4, %%T4, 4 ; shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%T1, %%T1, %%T4 ; the result is in %%T1 - - vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap - - vpxor %%XMM1, %%T1 - -%endmacro ; GHASH_8_ENCRYPT_8_PARALLEL - -; GHASH the last 4 ciphertext blocks. -%macro GHASH_LAST_8 16 -%define %%GDATA %1 -%define %%T1 %2 -%define %%T2 %3 -%define %%T3 %4 -%define %%T4 %5 -%define %%T5 %6 -%define %%T6 %7 -%define %%T7 %8 -%define %%XMM1 %9 -%define %%XMM2 %10 -%define %%XMM3 %11 -%define %%XMM4 %12 -%define %%XMM5 %13 -%define %%XMM6 %14 -%define %%XMM7 %15 -%define %%XMM8 %16 - - ;; Karatsuba Method - - vmovdqu %%T5, [%%GDATA + HashKey_8] - - vpshufd %%T2, %%XMM1, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM1 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T6, %%XMM1, %%T5, 0x11 - vpclmulqdq %%T7, %%XMM1, %%T5, 0x00 - - vpclmulqdq %%XMM1, %%T2, %%T3, 0x00 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_7] - vpshufd %%T2, %%XMM2, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM2 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM2, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM2, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_6] - vpshufd %%T2, %%XMM3, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM3 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM3, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM3, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_5] - vpshufd %%T2, %%XMM4, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM4 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM4, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM4, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_4] - vpshufd %%T2, %%XMM5, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM5 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM5, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM5, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_3] - vpshufd %%T2, %%XMM6, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM6 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM6, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM6, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_2] - vpshufd %%T2, %%XMM7, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM7 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM7, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM7, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey] - vpshufd %%T2, %%XMM8, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM8 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM8, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM8, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - vpxor %%XMM1, %%XMM1, %%T6 - vpxor %%T2, %%XMM1, %%T7 - - vpslldq %%T4, %%T2, 8 - vpsrldq %%T2, %%T2, 8 - - vpxor %%T7, %%T7, %%T4 - vpxor %%T6, %%T6, %%T2 ; <%%T6:%%T7> holds the result of the accumulated carry-less multiplications - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%T3, [rel POLY2] - - vpclmulqdq %%T2, %%T3, %%T7, 0x01 - vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs - - vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - ;second phase of the reduction - vpclmulqdq %%T2, %%T3, %%T7, 0x00 - vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%T4, %%T3, %%T7, 0x10 - vpslldq %%T4, %%T4, 4 ; shift-L %%T4 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%T6, %%T6, %%T4 ; the result is in %%T6 -%endmacro - -; GHASH the last 4 ciphertext blocks. -%macro GHASH_LAST_7 15 -%define %%GDATA %1 -%define %%T1 %2 -%define %%T2 %3 -%define %%T3 %4 -%define %%T4 %5 -%define %%T5 %6 -%define %%T6 %7 -%define %%T7 %8 -%define %%XMM1 %9 -%define %%XMM2 %10 -%define %%XMM3 %11 -%define %%XMM4 %12 -%define %%XMM5 %13 -%define %%XMM6 %14 -%define %%XMM7 %15 - - ;; Karatsuba Method - - vmovdqu %%T5, [%%GDATA + HashKey_7] - - vpshufd %%T2, %%XMM1, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM1 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T6, %%XMM1, %%T5, 0x11 - vpclmulqdq %%T7, %%XMM1, %%T5, 0x00 - - vpclmulqdq %%XMM1, %%T2, %%T3, 0x00 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_6] - vpshufd %%T2, %%XMM2, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM2 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM2, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM2, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_5] - vpshufd %%T2, %%XMM3, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM3 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM3, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM3, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_4] - vpshufd %%T2, %%XMM4, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM4 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM4, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM4, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_3] - vpshufd %%T2, %%XMM5, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM5 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM5, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM5, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_2] - vpshufd %%T2, %%XMM6, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM6 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM6, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM6, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_1] - vpshufd %%T2, %%XMM7, 01001110b - vpshufd %%T3, %%T5, 01001110b - vpxor %%T2, %%T2, %%XMM7 - vpxor %%T3, %%T3, %%T5 - - vpclmulqdq %%T4, %%XMM7, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM7, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpxor %%XMM1, %%XMM1, %%T6 - vpxor %%T2, %%XMM1, %%T7 - - vpslldq %%T4, %%T2, 8 - vpsrldq %%T2, %%T2, 8 - - vpxor %%T7, %%T7, %%T4 - vpxor %%T6, %%T6, %%T2 ; <%%T6:%%T7> holds the result of the accumulated carry-less multiplications - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%T3, [rel POLY2] - - vpclmulqdq %%T2, %%T3, %%T7, 0x01 - vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs - - vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - ;second phase of the reduction - vpclmulqdq %%T2, %%T3, %%T7, 0x00 - vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%T4, %%T3, %%T7, 0x10 - vpslldq %%T4, %%T4, 4 ; shift-L %%T4 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%T6, %%T6, %%T4 ; the result is in %%T6 -%endmacro - -;;; Handle encryption of the final partial block -;;; IN: -;;; r13 - Number of bytes to read -;;; MODIFIES: -;;; KEY - Key for encrypting the partial block -;;; HASH - Current hash value -;;; SMASHES: -;;; r10, r12, r15, rax -;;; T1, T2 -;;; Note: -;;; PLAIN_CYPH_LEN, %7, is passed only to determine -;;; if buffer is big enough to do a 16 byte read & shift. -;;; 'LT16' is passed here only if buffer is known to be smaller -;;; than 16 bytes. -;;; Any other value passed here will result in 16 byte read -;;; code path. -;;; TBD: Remove HASH from the instantiation -%macro ENCRYPT_FINAL_PARTIAL_BLOCK 8 -%define %%KEY %1 -%define %%T1 %2 -%define %%T2 %3 -%define %%CYPH_PLAIN_OUT %4 -%define %%PLAIN_CYPH_IN %5 -%define %%PLAIN_CYPH_LEN %6 -%define %%ENC_DEC %7 -%define %%DATA_OFFSET %8 - - ;; NOTE: type of read tuned based %%PLAIN_CYPH_LEN setting -%ifidn %%PLAIN_CYPH_LEN, LT16 - ;; Handle the case where the message is < 16 bytes - lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - - ;; T1 - packed output - ;; r10 - input data address - ;; r13 - input data length - ;; r12, r15, rax - temp registers - READ_SMALL_DATA_INPUT_AVX %%T1, r10, r13, r12, r15, rax - - lea r12, [SHIFT_MASK + 16] - sub r12, r13 -%else - ;; Handle the case where the message is >= 16 bytes - sub %%DATA_OFFSET, 16 - add %%DATA_OFFSET, r13 - ;; Receive the last <16 Byte block - vmovdqu %%T1, [%%PLAIN_CYPH_IN+%%DATA_OFFSET] - sub %%DATA_OFFSET, r13 - add %%DATA_OFFSET, 16 - - lea r12, [SHIFT_MASK + 16] - ;; Adjust the shuffle mask pointer to be able to shift 16-r13 bytes - ;; (r13 is the number of bytes in plaintext mod 16) - sub r12, r13 - ;; Get the appropriate shuffle mask - vmovdqu %%T2, [r12] - ;; shift right 16-r13 bytes - vpshufb %%T1, %%T2 -%endif ; %%PLAIN_CYPH_LEN, LT16 - - ;; At this point T1 contains the partial block data -%ifidn %%ENC_DEC, DEC - ;; Plaintext XOR E(K, Yn) - ;; Set aside the ciphertext - vmovdqa %%T2, %%T1 - vpxor %%KEY, %%KEY, %%T1 - ;; Get the appropriate mask to mask out top 16-r13 bytes of ciphertext - vmovdqu %%T1, [r12 + ALL_F - SHIFT_MASK] - ;; Mask out top 16-r13 bytes of ciphertext - vpand %%KEY, %%KEY, %%T1 - - ;; Prepare the ciphertext for the hash - ;; mask out top 16-r13 bytes of the plaintext - vpand %%T2, %%T2, %%T1 -%else - ;; Plaintext XOR E(K, Yn) - vpxor %%KEY, %%KEY, %%T1 - ;; Get the appropriate mask to mask out top 16-r13 bytes of %%KEY - vmovdqu %%T1, [r12 + ALL_F - SHIFT_MASK] - ;; Mask out top 16-r13 bytes of %%KEY - vpand %%KEY, %%KEY, %%T1 -%endif - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;; Output r13 Bytes - vmovq rax, %%KEY - cmp r13, 8 - jle %%_less_than_8_bytes_left - - mov [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], rax - add %%DATA_OFFSET, 8 - vpsrldq %%T1, %%KEY, 8 - vmovq rax, %%T1 - sub r13, 8 - -%%_less_than_8_bytes_left: - mov BYTE [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], al - add %%DATA_OFFSET, 1 - shr rax, 8 - sub r13, 1 - jne %%_less_than_8_bytes_left - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%ifidn %%ENC_DEC, DEC - ;; If decrypt, restore the ciphertext into %%KEY - vmovdqu %%KEY, %%T2 -%endif -%endmacro ; ENCRYPT_FINAL_PARTIAL_BLOCK - -; Encryption of a single block -%macro ENCRYPT_SINGLE_BLOCK 2 -%define %%GDATA %1 -%define %%XMM0 %2 - - vpxor %%XMM0, %%XMM0, [%%GDATA+16*0] -%assign i 1 -%rep NROUNDS - vaesenc %%XMM0, [%%GDATA+16*i] -%assign i (i+1) -%endrep - vaesenclast %%XMM0, [%%GDATA+16*i] -%endmacro - -;; Start of Stack Setup - -%macro FUNC_SAVE 0 - ;; Required for Update/GMC_ENC - ;the number of pushes must equal STACK_OFFSET - push r12 - push r13 - push r14 - push r15 - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 - -%ifidn __OUTPUT_FORMAT__, win64 - ; xmm6:xmm15 need to be maintained for Windows - vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 - vmovdqu [rsp + LOCAL_STORAGE + 1*16],xmm7 - vmovdqu [rsp + LOCAL_STORAGE + 2*16],xmm8 - vmovdqu [rsp + LOCAL_STORAGE + 3*16],xmm9 - vmovdqu [rsp + LOCAL_STORAGE + 4*16],xmm10 - vmovdqu [rsp + LOCAL_STORAGE + 5*16],xmm11 - vmovdqu [rsp + LOCAL_STORAGE + 6*16],xmm12 - vmovdqu [rsp + LOCAL_STORAGE + 7*16],xmm13 - vmovdqu [rsp + LOCAL_STORAGE + 8*16],xmm14 - vmovdqu [rsp + LOCAL_STORAGE + 9*16],xmm15 -%endif -%endmacro - -%macro FUNC_RESTORE 0 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_ymms_asm -%endif -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm15, [rsp + LOCAL_STORAGE + 9*16] - vmovdqu xmm14, [rsp + LOCAL_STORAGE + 8*16] - vmovdqu xmm13, [rsp + LOCAL_STORAGE + 7*16] - vmovdqu xmm12, [rsp + LOCAL_STORAGE + 6*16] - vmovdqu xmm11, [rsp + LOCAL_STORAGE + 5*16] - vmovdqu xmm10, [rsp + LOCAL_STORAGE + 4*16] - vmovdqu xmm9, [rsp + LOCAL_STORAGE + 3*16] - vmovdqu xmm8, [rsp + LOCAL_STORAGE + 2*16] - vmovdqu xmm7, [rsp + LOCAL_STORAGE + 1*16] - vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] -%endif - -;; Required for Update/GMC_ENC - mov rsp, r14 - pop r15 - pop r14 - pop r13 - pop r12 -%endmacro - -%macro CALC_J0 15 -%define %%KEY %1 ;; [in] Pointer to GCM KEY structure -%define %%IV %2 ;; [in] Pointer to IV -%define %%IV_LEN %3 ;; [in] IV length -%define %%J0 %4 ;; [out] XMM reg to contain J0 -%define %%TMP0 %5 ;; [clobbered] Temporary GP reg -%define %%TMP1 %6 ;; [clobbered] Temporary GP reg -%define %%TMP2 %7 ;; [clobbered] Temporary GP reg -%define %%TMP3 %8 ;; [clobbered] Temporary GP reg -%define %%TMP4 %9 ;; [clobbered] Temporary GP reg -%define %%XTMP0 %10 ;; [clobbered] Temporary XMM reg -%define %%XTMP1 %11 ;; [clobbered] Temporary XMM reg -%define %%XTMP2 %12 ;; [clobbered] Temporary XMM reg -%define %%XTMP3 %13 ;; [clobbered] Temporary XMM reg -%define %%XTMP4 %14 ;; [clobbered] Temporary XMM reg -%define %%XTMP5 %15 ;; [clobbered] Temporary XMM reg - - ;; J0 = GHASH(IV || 0s+64 || len(IV)64) - ;; s = 16 * RoundUp(len(IV)/16) - len(IV) */ - - ;; Calculate GHASH of (IV || 0s) - vpxor %%J0, %%J0 - CALC_AAD_HASH %%IV, %%IV_LEN, %%J0, %%KEY, %%XTMP0, %%XTMP1, %%XTMP2, \ - %%XTMP3, %%XTMP4, %%XTMP5, %%TMP0, %%TMP1, %%TMP2, %%TMP3, %%TMP4 - - ;; Calculate GHASH of last 16-byte block (0 || len(IV)64) - vmovdqu %%XTMP0, [%%KEY + HashKey] - mov %%TMP2, %%IV_LEN - shl %%TMP2, 3 ;; IV length in bits - vmovq %%XTMP1, %%TMP2 - vpxor %%J0, %%XTMP1 - GHASH_MUL %%J0, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 - - vpshufb %%J0, [rel SHUF_MASK] ; perform a 16Byte swap -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GCM_INIT initializes a gcm_context_data struct to prepare for encoding/decoding. -; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), IV, IV_LEN, -; Additional Authentication data (A_IN), Additional Data length (A_LEN). -; Output: Updated GDATA_CTX with the hash of A_IN (AadHash) and initialized other parts of GDATA. -; Clobbers rax, r10-r13 and xmm0-xmm6 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GCM_INIT 5-6 -%define %%GDATA_KEY %1 ; [in] Pointer to GCM Key data structure -%define %%GDATA_CTX %2 ; [in/out] Pointer to GCM Context data structure -%define %%IV %3 ; [in] Pointer to IV -%define %%A_IN %4 ; [in] Pointer to AAD -%define %%A_LEN %5 ; [in] AAD length -%define %%IV_LEN %6 ; [in] IV length - -%define %%AAD_HASH xmm14 - - mov r10, %%A_LEN - cmp r10, 0 - je %%_aad_is_zero - - vpxor %%AAD_HASH, %%AAD_HASH - CALC_AAD_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, r10, r11, r12, r13, rax - jmp %%_after_aad - -%%_aad_is_zero: - vpxor %%AAD_HASH, %%AAD_HASH - -%%_after_aad: - mov r10, %%A_LEN - vpxor xmm2, xmm3 - - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH ; ctx_data.aad hash = aad_hash - mov [%%GDATA_CTX + AadLen], r10 ; ctx_data.aad_length = aad_length - xor r10, r10 - mov [%%GDATA_CTX + InLen], r10 ; ctx_data.in_length = 0 - mov [%%GDATA_CTX + PBlockLen], r10 ; ctx_data.partial_block_length = 0 - vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm2 ; ctx_data.partial_block_enc_key = 0 - mov r10, %%IV -%if %0 == 6 ;; IV is different than 12 bytes - CALC_J0 %%GDATA_KEY, %%IV, %%IV_LEN, xmm2, r10, r11, r12, r13, rax, xmm1, xmm0, \ - xmm3, xmm4, xmm5, xmm6 -%else ;; IV is 12 bytes - vmovdqa xmm2, [rel ONEf] ; read 12 IV bytes and pad with 0x00000001 - vpinsrq xmm2, [r10], 0 - vpinsrd xmm2, [r10+8], 2 -%endif - vmovdqu [%%GDATA_CTX + OrigIV], xmm2 ; ctx_data.orig_IV = iv - - vpshufb xmm2, [rel SHUF_MASK] - - vmovdqu [%%GDATA_CTX + CurCount], xmm2 ; ctx_data.current_counter = iv -%endmacro - -%macro GCM_ENC_DEC_SMALL 12 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%CYPH_PLAIN_OUT %3 -%define %%PLAIN_CYPH_IN %4 -%define %%PLAIN_CYPH_LEN %5 -%define %%ENC_DEC %6 -%define %%DATA_OFFSET %7 -%define %%LENGTH %8 -%define %%NUM_BLOCKS %9 -%define %%CTR %10 -%define %%HASH %11 -%define %%INSTANCE_TYPE %12 - - ;; NOTE: the check below is obsolete in current implementation. The check is already done in GCM_ENC_DEC. - ;; cmp %%NUM_BLOCKS, 0 - ;; je %%_small_initial_blocks_encrypted - cmp %%NUM_BLOCKS, 8 - je %%_small_initial_num_blocks_is_8 - cmp %%NUM_BLOCKS, 7 - je %%_small_initial_num_blocks_is_7 - cmp %%NUM_BLOCKS, 6 - je %%_small_initial_num_blocks_is_6 - cmp %%NUM_BLOCKS, 5 - je %%_small_initial_num_blocks_is_5 - cmp %%NUM_BLOCKS, 4 - je %%_small_initial_num_blocks_is_4 - cmp %%NUM_BLOCKS, 3 - je %%_small_initial_num_blocks_is_3 - cmp %%NUM_BLOCKS, 2 - je %%_small_initial_num_blocks_is_2 - - jmp %%_small_initial_num_blocks_is_1 - -%%_small_initial_num_blocks_is_8: - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 8, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - jmp %%_small_initial_blocks_encrypted - -%%_small_initial_num_blocks_is_7: - ;; r13 - %%LENGTH - ;; xmm12 - T1 - ;; xmm13 - T2 - ;; xmm14 - T3 - AAD HASH OUT when not producing 8 AES keys - ;; xmm15 - T4 - ;; xmm11 - T5 - ;; xmm9 - CTR - ;; xmm1 - XMM1 - Cipher + Hash when producing 8 AES keys - ;; xmm2 - XMM2 - ;; xmm3 - XMM3 - ;; xmm4 - XMM4 - ;; xmm5 - XMM5 - ;; xmm6 - XMM6 - ;; xmm7 - XMM7 - ;; xmm8 - XMM8 - AAD HASH IN - ;; xmm10 - T6 - ;; xmm0 - T_key - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 7, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - jmp %%_small_initial_blocks_encrypted - -%%_small_initial_num_blocks_is_6: - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 6, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - jmp %%_small_initial_blocks_encrypted - -%%_small_initial_num_blocks_is_5: - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 5, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - jmp %%_small_initial_blocks_encrypted - -%%_small_initial_num_blocks_is_4: - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 4, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - jmp %%_small_initial_blocks_encrypted - -%%_small_initial_num_blocks_is_3: - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 3, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - jmp %%_small_initial_blocks_encrypted - -%%_small_initial_num_blocks_is_2: - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 2, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - jmp %%_small_initial_blocks_encrypted - -%%_small_initial_num_blocks_is_1: - INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 1, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE - - ;; Note: zero initial blocks not allowed. - -%%_small_initial_blocks_encrypted: - -%endmacro ; GCM_ENC_DEC_SMALL - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context_data struct -; has been initialized by GCM_INIT -; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. -; Input: gcm_key_data struct* (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), -; input text length (PLAIN_CYPH_LEN) and whether encoding or decoding (ENC_DEC). -; Output: A cypher of the given plain text (CYPH_PLAIN_OUT), and updated GDATA_CTX -; Clobbers rax, r10-r15, and xmm0-xmm15 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GCM_ENC_DEC 7 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%CYPH_PLAIN_OUT %3 -%define %%PLAIN_CYPH_IN %4 -%define %%PLAIN_CYPH_LEN %5 -%define %%ENC_DEC %6 -%define %%INSTANCE_TYPE %7 -%define %%DATA_OFFSET r11 - -; Macro flow: -; calculate the number of 16byte blocks in the message -; process (number of 16byte blocks) mod 8 '%%_initial_num_blocks_is_# .. %%_initial_blocks_encrypted' -; process 8 16 byte blocks at a time until all are done '%%_encrypt_by_8_new .. %%_eight_cipher_left' -; if there is a block of less than 16 bytes process it '%%_zero_cipher_left .. %%_multiple_of_16_bytes' - - cmp %%PLAIN_CYPH_LEN, 0 - je %%_enc_dec_done - - xor %%DATA_OFFSET, %%DATA_OFFSET - ;; Update length of data processed -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_CYPH_LEN - add [%%GDATA_CTX + InLen], rax -%else - add [%%GDATA_CTX + InLen], %%PLAIN_CYPH_LEN -%endif - vmovdqu xmm13, [%%GDATA_KEY + HashKey] - vmovdqu xmm8, [%%GDATA_CTX + AadHash] - -%ifidn %%INSTANCE_TYPE, multi_call - ;; NOTE: partial block processing makes only sense for multi_call here. - ;; Used for the update flow - if there was a previous partial - ;; block fill the remaining bytes here. - PARTIAL_BLOCK %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%DATA_OFFSET, xmm8, xmm13, %%ENC_DEC -%endif - - ;; lift CTR set from initial_blocks to here -%ifidn %%INSTANCE_TYPE, single_call - vmovdqu xmm9, xmm2 -%else - vmovdqu xmm9, [%%GDATA_CTX + CurCount] -%endif - - ;; Save the amount of data left to process in r10 - mov r13, %%PLAIN_CYPH_LEN -%ifidn %%INSTANCE_TYPE, multi_call - ;; NOTE: %%DATA_OFFSET is zero in single_call case. - ;; Consequently PLAIN_CYPH_LEN will never be zero after - ;; %%DATA_OFFSET subtraction below. - sub r13, %%DATA_OFFSET - - ;; There may be no more data if it was consumed in the partial block. - cmp r13, 0 - je %%_enc_dec_done -%endif ; %%INSTANCE_TYPE, multi_call - mov r10, r13 - - ;; Determine how many blocks to process in INITIAL - mov r12, r13 - shr r12, 4 - and r12, 7 - - ;; Process one additional block in INITIAL if there is a partial block - and r10, 0xf - blsmsk r10, r10 ; Set CF if zero - cmc ; Flip CF - adc r12, 0x0 ; Process an additional INITIAL block if CF set - - ;; Less than 127B will be handled by the small message code, which - ;; can process up to 7 16B blocks. - cmp r13, 128 - jge %%_large_message_path - - GCM_ENC_DEC_SMALL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET, r13, r12, xmm9, xmm14, %%INSTANCE_TYPE - jmp %%_ghash_done - -%%_large_message_path: - and r12, 0x7 ; Still, don't allow 8 INITIAL blocks since this will - ; can be handled by the x8 partial loop. - - cmp r12, 0 - je %%_initial_num_blocks_is_0 - cmp r12, 7 - je %%_initial_num_blocks_is_7 - cmp r12, 6 - je %%_initial_num_blocks_is_6 - cmp r12, 5 - je %%_initial_num_blocks_is_5 - cmp r12, 4 - je %%_initial_num_blocks_is_4 - cmp r12, 3 - je %%_initial_num_blocks_is_3 - cmp r12, 2 - je %%_initial_num_blocks_is_2 - - jmp %%_initial_num_blocks_is_1 - -%%_initial_num_blocks_is_7: - ;; r13 - %%LENGTH - ;; xmm12 - T1 - ;; xmm13 - T2 - ;; xmm14 - T3 - AAD HASH OUT when not producing 8 AES keys - ;; xmm15 - T4 - ;; xmm11 - T5 - ;; xmm9 - CTR - ;; xmm1 - XMM1 - Cipher + Hash when producing 8 AES keys - ;; xmm2 - XMM2 - ;; xmm3 - XMM3 - ;; xmm4 - XMM4 - ;; xmm5 - XMM5 - ;; xmm6 - XMM6 - ;; xmm7 - XMM7 - ;; xmm8 - XMM8 - AAD HASH IN - ;; xmm10 - T6 - ;; xmm0 - T_key - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 7, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_6: - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 6, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_5: - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 5, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_4: - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 4, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_3: - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 3, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_2: - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 2, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_1: - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 1, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_0: - INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 0, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - -%%_initial_blocks_encrypted: - ;; The entire message was encrypted processed in initial and now need to be hashed - cmp r13, 0 - je %%_encrypt_done - - ;; Encrypt the final <16 byte (partial) block, then hash - cmp r13, 16 - jl %%_encrypt_final_partial - - ;; Process 7 full blocks plus a partial block - cmp r13, 128 - jl %%_encrypt_by_8_partial - -%%_encrypt_by_8_parallel: - ;; in_order vs. out_order is an optimization to increment the counter without shuffling - ;; it back into little endian. r15d keeps track of when we need to increent in order so - ;; that the carry is handled correctly. - vmovd r15d, xmm9 - and r15d, 255 - vpshufb xmm9, [rel SHUF_MASK] - -%%_encrypt_by_8_new: - cmp r15d, 255-8 - jg %%_encrypt_by_8 - - ;; xmm0 - T1 - ;; xmm10 - T2 - ;; xmm11 - T3 - ;; xmm12 - T4 - ;; xmm13 - T5 - ;; xmm14 - T6 - ;; xmm9 - CTR - ;; xmm1 - XMM1 - ;; xmm2 - XMM2 - ;; xmm3 - XMM3 - ;; xmm4 - XMM4 - ;; xmm5 - XMM5 - ;; xmm6 - XMM6 - ;; xmm7 - XMM7 - ;; xmm8 - XMM8 - ;; xmm15 - T7 - add r15b, 8 - GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, out_order, %%ENC_DEC, full - add %%DATA_OFFSET, 128 - sub r13, 128 - cmp r13, 128 - jge %%_encrypt_by_8_new - - vpshufb xmm9, [rel SHUF_MASK] - jmp %%_encrypt_by_8_parallel_done - -%%_encrypt_by_8: - vpshufb xmm9, [rel SHUF_MASK] - add r15b, 8 - GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, in_order, %%ENC_DEC, full - vpshufb xmm9, [rel SHUF_MASK] - add %%DATA_OFFSET, 128 - sub r13, 128 - cmp r13, 128 - jge %%_encrypt_by_8_new - vpshufb xmm9, [rel SHUF_MASK] - -%%_encrypt_by_8_parallel_done: - ;; Test to see if we need a by 8 with partial block. At this point - ;; bytes remaining should be either zero or between 113-127. - cmp r13, 0 - je %%_encrypt_done - -%%_encrypt_by_8_partial: - ;; Shuffle needed to align key for partial block xor. out_order - ;; is a little faster because it avoids extra shuffles. - ;; TBD: Might need to account for when we don't have room to increment the counter. - - ;; Process parallel buffers with a final partial block. - GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, in_order, %%ENC_DEC, partial - - add %%DATA_OFFSET, 128-16 - sub r13, 128-16 - -%%_encrypt_final_partial: - - vpshufb xmm8, [rel SHUF_MASK] - mov [%%GDATA_CTX + PBlockLen], r13 - vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm8 - - ;; xmm8 - Final encrypted counter - need to hash with partial or full block ciphertext - ;; GDATA, KEY, T1, T2 - ENCRYPT_FINAL_PARTIAL_BLOCK xmm8, xmm0, xmm10, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET - - vpshufb xmm8, [rel SHUF_MASK] - -%%_encrypt_done: - - ;; Mapping to macro parameters - ;; IN: - ;; xmm9 contains the counter - ;; xmm1-xmm8 contain the xor'd ciphertext - ;; OUT: - ;; xmm14 contains the final hash - ;; GDATA, T1, T2, T3, T4, T5, T6, T7, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 -%ifidn %%INSTANCE_TYPE, multi_call - mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 - jz %%_hash_last_8 - GHASH_LAST_7 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 - ;; XOR the partial word into the hash - vpxor xmm14, xmm14, xmm8 - jmp %%_ghash_done -%endif -%%_hash_last_8: - GHASH_LAST_8 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 - -%%_ghash_done: - vmovdqu [%%GDATA_CTX + CurCount], xmm9 ; my_ctx_data.current_counter = xmm9 - vmovdqu [%%GDATA_CTX + AadHash], xmm14 ; my_ctx_data.aad hash = xmm14 - -%%_enc_dec_done: - -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GCM_COMPLETE Finishes Encryption/Decryption of last partial block after GCM_UPDATE finishes. -; Input: A gcm_key_data * (GDATA_KEY), gcm_context_data (GDATA_CTX). -; Output: Authorization Tag (AUTH_TAG) and Authorization Tag length (AUTH_TAG_LEN) -; Clobbers rax, r10-r12, and xmm0-xmm2, xmm5-xmm6, xmm9-xmm11, xmm13-xmm15 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GCM_COMPLETE 5 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%AUTH_TAG %3 -%define %%AUTH_TAG_LEN %4 -%define %%INSTANCE_TYPE %5 -%define %%PLAIN_CYPH_LEN rax - - vmovdqu xmm13, [%%GDATA_KEY + HashKey] - ;; Start AES as early as possible - vmovdqu xmm9, [%%GDATA_CTX + OrigIV] ; xmm9 = Y0 - ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Y0) - -%ifidn %%INSTANCE_TYPE, multi_call - ;; If the GCM function is called as a single function call rather - ;; than invoking the individual parts (init, update, finalize) we - ;; can remove a write to read dependency on AadHash. - vmovdqu xmm14, [%%GDATA_CTX + AadHash] - - ;; Encrypt the final partial block. If we did this as a single call then - ;; the partial block was handled in the main GCM_ENC_DEC macro. - mov r12, [%%GDATA_CTX + PBlockLen] - cmp r12, 0 - - je %%_partial_done - - GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block - vmovdqu [%%GDATA_CTX + AadHash], xmm14 - -%%_partial_done: - -%endif - - mov r12, [%%GDATA_CTX + AadLen] ; r12 = aadLen (number of bytes) - mov %%PLAIN_CYPH_LEN, [%%GDATA_CTX + InLen] - - shl r12, 3 ; convert into number of bits - vmovq xmm15, r12 ; len(A) in xmm15 - - shl %%PLAIN_CYPH_LEN, 3 ; len(C) in bits (*128) - vmovq xmm1, %%PLAIN_CYPH_LEN - vpslldq xmm15, xmm15, 8 ; xmm15 = len(A)|| 0x0000000000000000 - vpxor xmm15, xmm15, xmm1 ; xmm15 = len(A)||len(C) - - vpxor xmm14, xmm15 - GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 - vpshufb xmm14, [rel SHUF_MASK] ; perform a 16Byte swap - - vpxor xmm9, xmm9, xmm14 - -%%_return_T: - mov r10, %%AUTH_TAG ; r10 = authTag - mov r11, %%AUTH_TAG_LEN ; r11 = auth_tag_len - - cmp r11, 16 - je %%_T_16 - - cmp r11, 12 - je %%_T_12 - - cmp r11, 8 - je %%_T_8 - - simd_store_avx r10, xmm9, r11, r12, rax - jmp %%_return_T_done -%%_T_8: - vmovq rax, xmm9 - mov [r10], rax - jmp %%_return_T_done -%%_T_12: - vmovq rax, xmm9 - mov [r10], rax - vpsrldq xmm9, xmm9, 8 - vmovd eax, xmm9 - mov [r10 + 8], eax - jmp %%_return_T_done -%%_T_16: - vmovdqu [r10], xmm9 - -%%_return_T_done: - -%ifdef SAFE_DATA - ;; Clear sensitive data from context structure - vpxor xmm0, xmm0 - vmovdqu [%%GDATA_CTX + AadHash], xmm0 - vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm0 -%endif -%endmacro ; GCM_COMPLETE - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_precomp_128_avx_gen4 / -; aes_gcm_precomp_192_avx_gen4 / -; aes_gcm_precomp_256_avx_gen4 -; (struct gcm_key_data *key_data) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(precomp,_),function,) -FN_NAME(precomp,_): - endbranch64 -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_precomp -%endif - - push r12 - push r13 - push r14 - push r15 - - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 ; align rsp to 64 bytes - -%ifidn __OUTPUT_FORMAT__, win64 - ; only xmm6 needs to be maintained - vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 -%endif - - vpxor xmm6, xmm6 - ENCRYPT_SINGLE_BLOCK arg1, xmm6 ; xmm6 = HashKey - - vpshufb xmm6, [rel SHUF_MASK] - ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; - vmovdqa xmm2, xmm6 - vpsllq xmm6, xmm6, 1 - vpsrlq xmm2, xmm2, 63 - vmovdqa xmm1, xmm2 - vpslldq xmm2, xmm2, 8 - vpsrldq xmm1, xmm1, 8 - vpor xmm6, xmm6, xmm2 - ;reduction - vpshufd xmm2, xmm1, 00100100b - vpcmpeqd xmm2, [rel TWOONE] - vpand xmm2, xmm2, [rel POLY] - vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vmovdqu [arg1 + HashKey], xmm6 ; store HashKey<<1 mod poly - - PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] -%endif - mov rsp, r14 - - pop r15 - pop r14 - pop r13 - pop r12 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_ymms_asm -%endif -exit_precomp: - - ret - -%ifdef SAFE_PARAM -error_precomp: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - - jmp exit_precomp -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_init_128_avx_gen4 / aes_gcm_init_192_avx_gen4 / aes_gcm_init_256_avx_gen4 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *iv, -; const u8 *aad, -; u64 aad_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(init,_),function,) -FN_NAME(init,_): - endbranch64 - push r12 - push r13 -%ifidn __OUTPUT_FORMAT__, win64 - push r14 - push r15 - mov r14, rsp - ; xmm6 needs to be maintained for Windows - sub rsp, 1*16 - vmovdqu [rsp + 0*16], xmm6 -%endif - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_init - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_init - - ;; Check IV != NULL - cmp arg3, 0 - jz error_init - - ;; Check if aad_len == 0 - cmp arg5, 0 - jz skip_aad_check_init - - ;; Check aad != NULL (aad_len != 0) - cmp arg4, 0 - jz error_init - -skip_aad_check_init: -%endif - GCM_INIT arg1, arg2, arg3, arg4, arg5 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_ymms_asm -%endif -exit_init: - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6 , [rsp + 0*16] - mov rsp, r14 - pop r15 - pop r14 -%endif - pop r13 - pop r12 - ret - -%ifdef SAFE_PARAM -error_init: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_IV - - ;; Check if aad_len == 0 - cmp arg5, 0 - jz skip_aad_check_error_init - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_init: - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_init -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_init_var_iv_128_avx_gen4 / aes_gcm_init_var_iv_192_avx_gen4 / -; aes_gcm_init_var_iv_256_avx_gen4 -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(init_var_iv,_),function,) -FN_NAME(init_var_iv,_): - endbranch64 - push r12 - push r13 -%ifidn __OUTPUT_FORMAT__, win64 - push r14 - push r15 - mov r14, rsp - ; xmm6 needs to be maintained for Windows - sub rsp, 1*16 - vmovdqu [rsp + 0*16], xmm6 -%endif - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_init_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_init_IV - - ;; Check IV != NULL - cmp arg3, 0 - jz error_init_IV - - ;; Check iv_len != 0 - cmp arg4, 0 - jz error_init_IV - - ;; Check if aad_len == 0 - cmp arg6, 0 - jz skip_aad_check_init_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg5, 0 - jz error_init_IV - -skip_aad_check_init_IV: -%endif - cmp arg4, 12 - je iv_len_12_init_IV - - GCM_INIT arg1, arg2, arg3, arg5, arg6, arg4 - jmp skip_iv_len_12_init_IV - -iv_len_12_init_IV: - GCM_INIT arg1, arg2, arg3, arg5, arg6 - -skip_iv_len_12_init_IV: -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_ymms_asm -%endif -exit_init_IV: - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6 , [rsp + 0*16] - mov rsp, r14 - pop r15 - pop r14 -%endif - pop r13 - pop r12 - ret - -%ifdef SAFE_PARAM -error_init_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_IV - - ;; Check iv_len != 0 - IMB_ERR_CHECK_ZERO arg4, rax, IMB_ERR_IV_LEN - - ;; Check if aad_len == 0 - cmp arg6, 0 - jz skip_aad_check_error_init_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg5, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_init_IV: - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_init_IV -%endif - - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_update_avx_gen4 / aes_gcm_enc_192_update_avx_gen4 / -; aes_gcm_enc_128_update_avx_gen4 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc,_update_),function,) -FN_NAME(enc,_update_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_update_enc - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_update_enc - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz error_update_enc - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_update_enc - - ;; Check out != NULL (plaintext_len != 0) - cmp arg3, 0 - jz error_update_enc - - ;; Check in != NULL (plaintext_len != 0) - cmp arg4, 0 - jz error_update_enc -%endif - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, multi_call - -exit_update_enc: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_update_enc: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_update_enc - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (plaintext_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_update_enc: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_update_enc -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_update_avx_gen4 / aes_gcm_dec_192_update_avx_gen4 / -; aes_gcm_dec_256_update_avx_gen4 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec,_update_),function,) -FN_NAME(dec,_update_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_update_dec - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_update_dec - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz error_update_dec - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_update_dec - - ;; Check out != NULL (plaintext_len != 0) - cmp arg3, 0 - jz error_update_dec - - ;; Check in != NULL (plaintext_len != 0) - cmp arg4, 0 - jz error_update_dec -%endif - - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, multi_call - -exit_update_dec: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_update_dec: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_update_dec - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (plaintext_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_update_dec: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_update_dec -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_finalize_avx_gen4 / aes_gcm_enc_192_finalize_avx_gen4 / -; aes_gcm_enc_256_finalize_avx_gen4 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc,_finalize_),function,) -FN_NAME(enc,_finalize_): - endbranch64 -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc_fin - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc_fin - - ;; Check auth_tag != NULL - cmp arg3, 0 - jz error_enc_fin - - ;; Check auth_tag_len == 0 or > 16 - cmp arg4, 0 - jz error_enc_fin - - cmp arg4, 16 - ja error_enc_fin -%endif - push r12 - -%ifidn __OUTPUT_FORMAT__, win64 - ; xmm6:xmm15 need to be maintained for Windows - sub rsp, 7*16 - vmovdqu [rsp + 0*16], xmm6 - vmovdqu [rsp + 1*16], xmm9 - vmovdqu [rsp + 2*16], xmm10 - vmovdqu [rsp + 3*16], xmm11 - vmovdqu [rsp + 4*16], xmm13 - vmovdqu [rsp + 5*16], xmm14 - vmovdqu [rsp + 6*16], xmm15 -%endif - GCM_COMPLETE arg1, arg2, arg3, arg4, multi_call - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_ymms_asm -%endif -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm15, [rsp + 6*16] - vmovdqu xmm14, [rsp + 5*16] - vmovdqu xmm13, [rsp + 4*16] - vmovdqu xmm11, [rsp + 3*16] - vmovdqu xmm10, [rsp + 2*16] - vmovdqu xmm9, [rsp + 1*16] - vmovdqu xmm6, [rsp + 0*16] - add rsp, 7*16 -%endif - pop r12 -exit_enc_fin: - ret - -%ifdef SAFE_PARAM -error_enc_fin: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg4, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg4, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc_fin -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_finalize_avx_gen4 / aes_gcm_dec_192_finalize_avx_gen4 -; aes_gcm_dec_256_finalize_avx_gen4 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec,_finalize_),function,) -FN_NAME(dec,_finalize_): - endbranch64 -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec_fin - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec_fin - - ;; Check auth_tag != NULL - cmp arg3, 0 - jz error_dec_fin - - ;; Check auth_tag_len == 0 or > 16 - cmp arg4, 0 - jz error_dec_fin - - cmp arg4, 16 - ja error_dec_fin -%endif - - push r12 - -%ifidn __OUTPUT_FORMAT__, win64 - ; xmm6:xmm15 need to be maintained for Windows - sub rsp, 7*16 - vmovdqu [rsp + 0*16], xmm6 - vmovdqu [rsp + 1*16], xmm9 - vmovdqu [rsp + 2*16], xmm10 - vmovdqu [rsp + 3*16], xmm11 - vmovdqu [rsp + 4*16], xmm13 - vmovdqu [rsp + 5*16], xmm14 - vmovdqu [rsp + 6*16], xmm15 -%endif - GCM_COMPLETE arg1, arg2, arg3, arg4, multi_call - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_ymms_asm -%endif -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm15, [rsp + 6*16] - vmovdqu xmm14, [rsp + 5*16] - vmovdqu xmm13, [rsp + 4*16] - vmovdqu xmm11, [rsp + 3*16] - vmovdqu xmm10, [rsp + 2*16] - vmovdqu xmm9, [rsp + 1*16] - vmovdqu xmm6, [rsp + 0*16] - add rsp, 7*16 -%endif - - pop r12 - -exit_dec_fin: - ret - -%ifdef SAFE_PARAM -error_dec_fin: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg4, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg4, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec_fin -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_avx_gen4 / aes_gcm_enc_192_avx_gen4 / aes_gcm_enc_256_avx_gen4 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u8 *aad, -; u64 aad_len, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc,_),function,) -FN_NAME(enc,_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc - - ;; Check IV != NULL - cmp arg6, 0 - jz error_enc - - ;; Check auth_tag != NULL - cmp arg9, 0 - jz error_enc - - ;; Check auth_tag_len == 0 or > 16 - cmp arg10, 0 - jz error_enc - - cmp arg10, 16 - ja error_enc - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_enc - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_enc - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_enc - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_enc - -skip_in_out_check_enc: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_enc - - ;; Check aad != NULL (aad_len != 0) - cmp arg7, 0 - jz error_enc - -skip_aad_check_enc: -%endif - GCM_INIT arg1, arg2, arg6, arg7, arg8 - - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, single_call - - GCM_COMPLETE arg1, arg2, arg9, arg10, single_call - -exit_enc: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_enc: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg9, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg10, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg10, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_enc - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_enc: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_error_enc - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg7, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_enc: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_avx_gen4 / aes_gcm_dec_192_avx_gen4 / aes_gcm_dec_256_avx_gen4 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u8 *aad, -; u64 aad_len, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec,_),function,) -FN_NAME(dec,_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec - - ;; Check IV != NULL - cmp arg6, 0 - jz error_dec - - ;; Check auth_tag != NULL - cmp arg9, 0 - jz error_dec - - ;; Check auth_tag_len == 0 or > 16 - cmp arg10, 0 - jz error_dec - - cmp arg10, 16 - ja error_dec - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_dec - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_dec - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_dec - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_dec - -skip_in_out_check_dec: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_dec - - ;; Check aad != NULL (aad_len != 0) - cmp arg7, 0 - jz error_dec - -skip_aad_check_dec: -%endif - GCM_INIT arg1, arg2, arg6, arg7, arg8 - - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call - - GCM_COMPLETE arg1, arg2, arg9, arg10, single_call - -exit_dec: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_dec: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg9, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg10, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg10, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_dec - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_dec: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_error_dec - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg7, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_dec: - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_var_iv_128_avx_gen4 / aes_gcm_enc_var_iv_192_avx_gen4 / -; aes_gcm_enc_var_iv_256_avx_gen4 -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc_var_iv,_),function,) -FN_NAME(enc_var_iv,_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_enc_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_enc_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_enc_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_enc_IV - - cmp arg11, 16 - ja error_enc_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_enc_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_enc_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_enc_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_enc_IV - -skip_in_out_check_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_enc_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_enc_IV - -skip_aad_check_enc_IV: -%endif - cmp arg7, 12 - je iv_len_12_enc_IV - - GCM_INIT arg1, arg2, arg6, arg8, arg9, arg7 - jmp skip_iv_len_12_enc_IV - -iv_len_12_enc_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9 - -skip_iv_len_12_enc_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, single_call - - GCM_COMPLETE arg1, arg2, arg10, arg11, single_call - -exit_enc_IV: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_enc_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_enc_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_enc_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_enc_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc_IV -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_var_iv_128_avx_gen4 / aes_gcm_dec_var_iv_192_avx_gen4 / -; aes_gcm_dec_var_iv_256_avx_gen4 -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec_var_iv,_),function,) -FN_NAME(dec_var_iv,_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_dec_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_dec_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_dec_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_dec_IV - - cmp arg11, 16 - ja error_dec_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_dec_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_dec_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_dec_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_dec_IV - -skip_in_out_check_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_dec_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_dec_IV - -skip_aad_check_dec_IV: -%endif - cmp arg7, 12 - je iv_len_12_dec_IV - - GCM_INIT arg1, arg2, arg6, arg8, arg9, arg7 - jmp skip_iv_len_12_dec_IV - -iv_len_12_dec_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9 - -skip_iv_len_12_dec_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call - - GCM_COMPLETE arg1, arg2, arg10, arg11, single_call - -exit_dec_IV: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_dec_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_dec_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_dec_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_dec_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec_IV -%endif - -%ifdef GCM128_MODE -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void ghash_avx_gen4 -; const struct gcm_key_data *key_data, -; const void *in, -; const u64 in_len, -; void *io_tag, -; const u64 tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(ghash_avx_gen4,function,) -ghash_avx_gen4: - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_ghash - - ;; Check in != NULL - cmp arg2, 0 - jz error_ghash - - ;; Check in_len != 0 - cmp arg3, 0 - jz error_ghash - - ;; Check tag != NULL - cmp arg4, 0 - jz error_ghash - - ;; Check tag_len != 0 - cmp arg5, 0 - jz error_ghash -%endif - - ;; copy tag to xmm0 - vmovdqu xmm0, [arg4] - vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap - - CALC_AAD_HASH arg2, arg3, xmm0, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ - r10, r11, r12, r13, rax - - vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap - - simd_store_avx arg4, xmm0, arg5, r12, rax - -exit_ghash: - FUNC_RESTORE - ret - -%ifdef SAFE_PARAM -error_ghash: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check in != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_SRC - - ;; Check in_len != 0 - IMB_ERR_CHECK_ZERO arg3, rax, IMB_ERR_AUTH_LEN - - ;; Check tag != NULL - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_AUTH - - ;; Check tag_len != 0 - IMB_ERR_CHECK_ZERO arg5, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - - jmp exit_ghash -%endif - -%endif ;; GCM128_MODE - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; PARTIAL_BLOCK_GMAC: Handles the tag partial blocks between update calls. -; Requires the input data be at least 1 byte long. -; Input: gcm_key_data (GDATA_KEY), gcm_context_data (GDATA_CTX), input text (PLAIN_IN), -; input text length (PLAIN_LEN), hash subkey (HASH_SUBKEY). -; Output: Updated GDATA_CTX -; Clobbers rax, r10, r12, r13, r15, xmm0, xmm1, xmm2, xmm3, xmm5, xmm6, xmm9, xmm10, xmm11, xmm13 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro PARTIAL_BLOCK_GMAC 7 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%PLAIN_IN %3 -%define %%PLAIN_LEN %4 -%define %%DATA_OFFSET %5 -%define %%AAD_HASH %6 -%define %%HASH_SUBKEY %7 - - mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 - ; Leave Macro if no partial blocks - je %%_partial_block_done - - ; Read in input data without over reading - cmp %%PLAIN_LEN, 16 - jl %%_fewer_than_16_bytes - ; If more than 16 bytes of data, just fill the xmm register - VXLDR xmm1, [%%PLAIN_IN] - jmp %%_data_read - -%%_fewer_than_16_bytes: - lea r10, [%%PLAIN_IN] - READ_SMALL_DATA_INPUT_AVX xmm1, r10, %%PLAIN_LEN, rax, r12, r15 - - ; Finished reading in data -%%_data_read: - - lea r12, [rel SHIFT_MASK] - ; Adjust the shuffle mask pointer to be able to shift r13 bytes - ; (16-r13 is the number of bytes in plaintext mod 16) - add r12, r13 - ; Get the appropriate shuffle mask - vmovdqu xmm2, [r12] - vmovdqa xmm3, xmm1 - - mov r15, %%PLAIN_LEN - add r15, r13 - ; Set r15 to be the amount of data left in PLAIN_IN after filling the block - sub r15, 16 - ; Determine if partial block is not being filled and shift mask accordingly - jge %%_no_extra_mask_1 - sub r12, r15 -%%_no_extra_mask_1: - - ; Get the appropriate mask to mask out bottom r13 bytes of xmm3 - vmovdqu xmm1, [r12 + ALL_F-SHIFT_MASK] - - vpand xmm3, xmm1 - vpshufb xmm3, [rel SHUF_MASK] - vpshufb xmm3, xmm2 - vpxor %%AAD_HASH, xmm3 - - cmp r15,0 - jl %%_partial_incomplete_1 - - ; GHASH computation for the last <16 Byte block - GHASH_MUL %%AAD_HASH, %%HASH_SUBKEY, xmm0, xmm10, xmm11, xmm5, xmm6 - xor rax, rax - mov [%%GDATA_CTX + PBlockLen], rax - jmp %%_ghash_done -%%_partial_incomplete_1: -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_LEN - add [%%GDATA_CTX + PBlockLen], rax -%else - add [%%GDATA_CTX + PBlockLen], %%PLAIN_LEN -%endif -%%_ghash_done: - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH - - cmp r15, 0 - jl %%_partial_fill - - mov r12, 16 - ; Set r12 to be the number of bytes to skip after this macro - sub r12, r13 - - jmp %%offset_set -%%_partial_fill: - mov r12, %%PLAIN_LEN -%%offset_set: - mov %%DATA_OFFSET, r12 -%%_partial_block_done: -%endmacro ; PARTIAL_BLOCK_GMAC - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void imb_aes_gmac_update_128_avx_gen4 / imb_aes_gmac_update_192_avx_gen4 / -; imb_aes_gmac_update_256_avx_gen4 -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; const u8 *in, -; const u64 msg_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(GMAC_FN_NAME(update),function,) -GMAC_FN_NAME(update): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET -%endif - ;; Check if msg_len == 0 - cmp arg4, 0 - je exit_gmac_update - -%ifdef SAFE_PARAM - ;; Check key_data != NULL - cmp arg1, 0 - jz error_gmac_update - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_gmac_update - - ;; Check in != NULL (msg_len != 0) - cmp arg3, 0 - jz error_gmac_update -%endif - - ; Increment size of "AAD length" for GMAC - add [arg2 + AadLen], arg4 - - ;; Deal with previous partial block - xor r11, r11 - vmovdqu xmm13, [arg1 + HashKey] - vmovdqu xmm8, [arg2 + AadHash] - - PARTIAL_BLOCK_GMAC arg1, arg2, arg3, arg4, r11, xmm8, xmm13 - - ; CALC_AAD_HASH needs to deal with multiple of 16 bytes - sub arg4, r11 - add arg3, r11 - - vmovq xmm7, arg4 ; Save remaining length - and arg4, -16 ; Get multiple of 16 bytes - - or arg4, arg4 - jz no_full_blocks - - ;; Calculate GHASH of this segment - CALC_AAD_HASH arg3, arg4, xmm8, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ - r10, r11, r12, r13, rax - vmovdqu [arg2 + AadHash], xmm8 ; ctx_data.aad hash = aad_hash - -no_full_blocks: - add arg3, arg4 ; Point at partial block - - vmovq arg4, xmm7 ; Restore original remaining length - and arg4, 15 - jz exit_gmac_update - - ; Save next partial block - mov [arg2 + PBlockLen], arg4 - READ_SMALL_DATA_INPUT_AVX xmm1, arg3, arg4, r11, r12, r13 - vpshufb xmm1, [rel SHUF_MASK] - vpxor xmm8, xmm1 - vmovdqu [arg2 + AadHash], xmm8 - -exit_gmac_update: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_gmac_update: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_SRC - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_gmac_update -%endif - -mksection stack-noexec diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 731658a87784c6e43f97b894566bdc69e0cf1f97..76ba8ed0903db04f981ad2a78e8d05f7c76fcdf0 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -53,11 +53,20 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) state->features = cpu_feature_adjust(state->flags, cpu_feature_detect()); +#ifdef AVX_IFMA + if ((state->features & IMB_CPUFLAGS_AVX2_T3) == + IMB_CPUFLAGS_AVX2_T3) { + init_mb_mgr_avx2_t3_internal(state, reset_mgrs); + return; + } +#endif if ((state->features & IMB_CPUFLAGS_AVX2_T2) == - IMB_CPUFLAGS_AVX2_T2) + IMB_CPUFLAGS_AVX2_T2) { init_mb_mgr_avx2_t2_internal(state, reset_mgrs); - else - init_mb_mgr_avx2_t1_internal(state, reset_mgrs); + return; + } + + init_mb_mgr_avx2_t1_internal(state, reset_mgrs); } void diff --git a/lib/avx2_t1/mb_mgr_avx2_t1.c b/lib/avx2_t1/mb_mgr_avx2_t1.c index 249c62d3f11cecaf80bbf9e398e6205390773f93..6bdcbcf60447851e04662883906437531ac56fd1 100644 --- a/lib/avx2_t1/mb_mgr_avx2_t1.c +++ b/lib/avx2_t1/mb_mgr_avx2_t1.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -71,6 +71,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2_t1 #define SUBMIT_HASH_BURST submit_hash_burst_avx2_t1 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx2_t1 +#define SET_SUITE_ID_FN set_suite_id_avx2_t1 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX2 @@ -359,6 +360,7 @@ init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_avx2; state->keyexp_192 = aes_keyexp_192_avx2; @@ -383,6 +385,7 @@ init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_avx2; state->aes128_cfb_one = aes_cfb_128_one_avx2; + state->aes256_cfb_one = aes_cfb_256_one_avx2; state->eea3_1_buffer = zuc_eea3_1_buffer_avx2; state->eea3_4_buffer = zuc_eea3_4_buffer_avx; @@ -467,7 +470,7 @@ init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs) state->gcm256_pre = aes_gcm_pre_256_avx_gen4; state->ghash = ghash_avx_gen4; - state->ghash_pre = ghash_pre_avx_gen2; + state->ghash_pre = ghash_pre_avx_gen4; state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; @@ -478,6 +481,9 @@ init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_avx; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_avx; } #include "mb_mgr_code.h" diff --git a/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm index 9583367c69a6aec36b9a0abe9e26917c31b74f70..55c1bd0628789f0089ea18781056262a2c5a2562 100644 --- a/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern md5_x8x2_avx2 mksection .rodata diff --git a/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm index 9ddc9305f22940f2ff28f50fb0096f99487a2cce..ea6736f835d04a67eeee97e5d3b6f0280bf40f7b 100644 --- a/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,14 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/memcpy.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/memcpy.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern md5_x8x2_avx2 %if 1 diff --git a/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm index 1dda74d9d14c03ec602e15f631350cc46adb0156..0bb10dc99e61501e5c09007d7be6041a1b8a195a 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha1_x8_avx2 mksection .rodata diff --git a/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm index d7dbea7377377f7c062314243924d0548d552beb..765875e1021f4b75166ad9b5903a1f943422d2b0 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,13 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" %include "include/const.inc" extern sha1_x8_avx2 diff --git a/lib/avx2_t1/mb_mgr_hmac_sha224_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha224_flush_avx2.asm index bae687b7111b0f5120a7d004847c19f2413d0fff..66845ef3cd989e9db2461294cc5b15a3ebc63f0a 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha224_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha224_flush_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx2_t1/mb_mgr_hmac_sha224_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha224_submit_avx2.asm index 0b6c7fdeee7825a9346f35e73a77e3667dd22303..2527060f13f4cbf5b44adc08ecd47e8a98918901 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha224_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha224_submit_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm index 7b6d3746e03c18746f07906b6166e1351ef4604a..ffac80f0ecb64b2cfc606f5525b979bc8d2ee81f 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern sha256_oct_avx2 diff --git a/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm index 30575b35aa4f1a278766e073fd1599d85a9a0354..45bf85ae05d4cca60eb2499d8d4faaf575063886 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" extern sha256_oct_avx2 diff --git a/lib/avx2_t1/mb_mgr_hmac_sha384_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha384_flush_avx2.asm index 9f4c13dd4e18f0ac0152f4a0fe27f26da05b1aa3..575756961b468e4c18040bf023ad2a100e1dd527 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha384_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha384_flush_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx2_t1/mb_mgr_hmac_sha384_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha384_submit_avx2.asm index a7687b52570700115207b78c660299d83a2c6f24..007991e160dc900ef178168c1caa8ec5f0e304b0 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha384_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha384_submit_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm index 1c13d9dce3e91ccaee92b423363e5b80af8e1b12..1009817ef06ff1c0d74097d448dc54238eee5dc1 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern sha512_x4_avx2 diff --git a/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm index 7b87b76343e95132002c5eec974b9293f48bed33..8ec61467fd4e77d2ba9892b48d17a328c4bfd91b 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" extern sha512_x4_avx2 diff --git a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm index b7f52fc3da4352d8f99cb5f893c18d7cde24b1e6..c0a6e79fe5eaff2bf91fee223c146e569c75640a 100644 --- a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,15 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" %include "include/cet.inc" -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" +%ifndef SUBMIT_JOB_ZUC128_EEA3 %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_avx2 %define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_avx2 %define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx2 @@ -43,6 +44,10 @@ %define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx2 %define ZUC128_INIT_8 asm_ZucInitialization_8_avx2 %define ZUC256_INIT_8 asm_Zuc256Initialization_8_avx2 +%define ZUC_EIA3_8_BUFFER zuc_eia3_8_buffer_job_avx2 +%define ZUC256_EIA3_8_BUFFER zuc256_eia3_8_buffer_job_avx2 +%define ZUC_CIPHER_8 asm_ZucCipher_8_avx2 +%endif mksection .rodata default rel @@ -80,11 +85,11 @@ dd 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF dd 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF dd 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF -extern zuc_eia3_8_buffer_job_avx2 -extern zuc256_eia3_8_buffer_job_avx2 -extern asm_ZucInitialization_8_avx2 -extern asm_Zuc256Initialization_8_avx2 -extern asm_ZucCipher_8_avx2 +extern ZUC128_INIT_8 +extern ZUC256_INIT_8 +extern ZUC_CIPHER_8 +extern ZUC_EIA3_8_BUFFER +extern ZUC256_EIA3_8_BUFFER %ifdef LINUX %define arg1 rdi @@ -382,7 +387,7 @@ mksection .text lea arg4, [r12 + _zuc_lens] mov arg5, min_len - call asm_ZucCipher_8_avx2 + call ZUC_CIPHER_8 RESTORE_STACK_SPACE 5 @@ -651,7 +656,7 @@ APPEND3(%%skip_eea3_copy_,I,J): lea arg4, [r12 + _zuc_lens] mov arg5, min_len - call asm_ZucCipher_8_avx2 + call ZUC_CIPHER_8 RESTORE_STACK_SPACE 5 @@ -878,9 +883,9 @@ FLUSH_JOB_ZUC256_EEA3: %endif %if %%KEY_SIZE == 128 - call zuc_eia3_8_buffer_job_avx2 + call ZUC_EIA3_8_BUFFER %else - call zuc256_eia3_8_buffer_job_avx2 + call ZUC256_EIA3_8_BUFFER %endif %if %%KEY_SIZE == 128 @@ -1046,9 +1051,9 @@ APPEND(%%skip_eia3_,I): %endif %if %%KEY_SIZE == 128 - call zuc_eia3_8_buffer_job_avx2 + call ZUC_EIA3_8_BUFFER %else - call zuc256_eia3_8_buffer_job_avx2 + call ZUC256_EIA3_8_BUFFER %endif %if %%KEY_SIZE == 128 diff --git a/lib/avx2_t1/md5_x8x2_avx2.asm b/lib/avx2_t1/md5_x8x2_avx2.asm index 19b90acb85d25b093eabcb20239b0e22910f5957..019be9c33aae3e6437a242b011828f8d3f8ff214 100644 --- a/lib/avx2_t1/md5_x8x2_avx2.asm +++ b/lib/avx2_t1/md5_x8x2_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -36,10 +36,10 @@ ;; ;; clobbers ymm0-15 -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/transpose_avx2.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/transpose_avx2.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/avx2_t1/sha1_x8_avx2.asm b/lib/avx2_t1/sha1_x8_avx2.asm index 5b0de3e54c0edc60586cfdbe5d52931c73233d9a..7f665cf70fe429f32620ae51ec8a320ee572d1c3 100644 --- a/lib/avx2_t1/sha1_x8_avx2.asm +++ b/lib/avx2_t1/sha1_x8_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -34,12 +34,12 @@ ;; ;; clobbers ymm0-15 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/transpose_avx2.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/transpose_avx2.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/avx2_t1/sha256_oct_avx2.asm b/lib/avx2_t1/sha256_oct_avx2.asm index f56b7256e7fa3fd08592a7e9e74816b314783d8a..d6f7c5310b4e73682bf665706696e5fc7327a65d 100644 --- a/lib/avx2_t1/sha256_oct_avx2.asm +++ b/lib/avx2_t1/sha256_oct_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -39,12 +39,12 @@ ;; ;; clobbers ymm0-15 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/transpose_avx2.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/transpose_avx2.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/avx2_t1/sha512_x4_avx2.asm b/lib/avx2_t1/sha512_x4_avx2.asm index f3c042e9422c3631a736e5344bf060ea0b795b97..db5258f8b288294207f7ee747e53e9a9b58e06dc 100644 --- a/lib/avx2_t1/sha512_x4_avx2.asm +++ b/lib/avx2_t1/sha512_x4_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -40,13 +40,12 @@ ;; ;; clobbers ymm0-15 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/transpose_avx2.asm" -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/transpose_avx2.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" mksection .rodata default rel align 64 diff --git a/lib/avx2_t1/sha_avx2.c b/lib/avx2_t1/sha_avx2.c new file mode 100644 index 0000000000000000000000000000000000000000..69f77b750756fbebeb4604e00631b8a205c5162f --- /dev/null +++ b/lib/avx2_t1/sha_avx2.c @@ -0,0 +1,115 @@ +/******************************************************************************* + Copyright (c) 2020-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_generic.h" +#include "include/arch_avx2_type1.h" + +/* ========================================================================== */ +/* One block SHA1 computation for IPAD / OPAD usage only */ +void sha1_one_block_avx2(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 1 /* SHA1 */); +} + +/* ========================================================================== */ +/* + * SHA1 API for use in HMAC-SHA1 when key is longer than the block size + */ +void sha1_avx2(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 1, IMB_SHA1_BLOCK_SIZE, + SHA1_PAD_SIZE); +} + +/* ========================================================================== */ +/* One block SHA224 computation for IPAD / OPAD usage only */ +void sha224_one_block_avx2(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 224 /* SHA224 */); +} + +/* ========================================================================== */ +/* + * SHA224 API for use in HMAC-SHA224 when key is longer than the block size + */ +void sha224_avx2(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE); +} + +/* ========================================================================== */ +/* One block SHA256 computation for IPAD / OPAD usage only */ +void sha256_one_block_avx2(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 256 /* SHA256 */); +} + +/* ========================================================================== */ +/* + * SHA256 API for use in HMAC-SHA256 when key is longer than the block size + */ +void sha256_avx2(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE); +} + + +/* ========================================================================== */ +/* One block SHA384 computation for IPAD / OPAD usage only */ +void sha384_one_block_avx2(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 384 /* SHA384 */); +} + +/* ========================================================================== */ +/* + * SHA384 API for use in HMAC-SHA384 when key is longer than the block size + */ +void sha384_avx2(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 384, + IMB_SHA_384_BLOCK_SIZE, SHA384_PAD_SIZE); +} + +/* ========================================================================== */ +/* One block SHA512 computation for IPAD / OPAD usage only */ +void sha512_one_block_avx2(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 512 /* SHA512 */); +} + +/* ========================================================================== */ +/* + * SHA512 API for use in HMAC-SHA512 when key is longer than the block size + */ +void sha512_avx2(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE); +} diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index fcca302151f95342255d81affacf83ddcab6248a..e97193d84c519418b72314ef85083a63a96a9152 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx2_t1/snow3g_avx2.c b/lib/avx2_t1/snow3g_avx2.c index 82f1976efd60a9d1a721c7a99310576340d555b3..8daad666c321664e5791ef426ea96e302d0a16e0 100644 --- a/lib/avx2_t1/snow3g_avx2.c +++ b/lib/avx2_t1/snow3g_avx2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx2_t1/zuc_top_avx2.c b/lib/avx2_t1/zuc_top_avx2.c index 91cdfedd6f4608cbd70300fc42878fe3bf62bf15..9e0f8ed25a20d7c3810bfb37898b80919cf49509 100644 --- a/lib/avx2_t1/zuc_top_avx2.c +++ b/lib/avx2_t1/zuc_top_avx2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -50,6 +50,51 @@ #define NUM_AVX2_BUFS 8 +static inline void +init_8(ZucKey8_t *keys, const uint8_t *ivs, ZucState8_t *state, + const uint64_t key_sz, const uint64_t tag_sz, + void *T, const unsigned use_gfni) +{ + if (key_sz == 128) { + if (use_gfni) + asm_ZucInitialization_8_gfni_avx2(keys, ivs, state); + else + asm_ZucInitialization_8_avx2(keys, ivs, state); + } else { + if (use_gfni) + asm_Zuc256Initialization_8_gfni_avx2(keys, ivs, + state, T, tag_sz); + else + asm_Zuc256Initialization_8_avx2(keys, ivs, state, T, + tag_sz); + } +} + +static inline void +keygen_8(ZucState8_t *state, uint32_t **pKeyStrArr, + const uint64_t numKeyStrBytes, const unsigned use_gfni) +{ + if (use_gfni) { + if (numKeyStrBytes == 4) + asm_ZucGenKeystream4B_8_gfni_avx2(state, pKeyStrArr); + else if (numKeyStrBytes == 8) + asm_ZucGenKeystream8B_8_gfni_avx2(state, pKeyStrArr); + else if (numKeyStrBytes == 16) + asm_ZucGenKeystream16B_8_gfni_avx2(state, pKeyStrArr); + else /* 32 */ + asm_ZucGenKeystream32B_8_gfni_avx2(state, pKeyStrArr); + } else { + if (numKeyStrBytes == 4) + asm_ZucGenKeystream4B_8_avx2(state, pKeyStrArr); + else if (numKeyStrBytes == 8) + asm_ZucGenKeystream8B_8_avx2(state, pKeyStrArr); + else if (numKeyStrBytes == 16) + asm_ZucGenKeystream16B_8_avx2(state, pKeyStrArr); + else /* 32 */ + asm_ZucGenKeystream32B_8_avx2(state, pKeyStrArr); + } +} + static inline uint16_t find_min_length16(const uint16_t length[NUM_AVX2_BUFS], unsigned int *allCommonBits) @@ -713,12 +758,14 @@ void zuc_eia3_1_buffer_avx2(const void *pKey, #endif } -void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], - const uint8_t *ivs, - const void * const pBufferIn[NUM_AVX2_BUFS], - uint32_t *pMacI[NUM_AVX2_BUFS], - const uint16_t lengthInBits[NUM_AVX2_BUFS], - const void * const job_in_lane[NUM_AVX2_BUFS]) +static inline +void _zuc_eia3_8_buffer_job(const void * const pKey[NUM_AVX2_BUFS], + const uint8_t *ivs, + const void * const pBufferIn[NUM_AVX2_BUFS], + uint32_t *pMacI[NUM_AVX2_BUFS], + const uint16_t lengthInBits[NUM_AVX2_BUFS], + const void * const job_in_lane[NUM_AVX2_BUFS], + const unsigned use_gfni) { unsigned int i = 0; DECLARE_ALIGNED(ZucState8_t state, 64); @@ -742,10 +789,10 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], keys.pKeys[i] = pKey[i]; } - asm_ZucInitialization_8_avx2(&keys, ivs, &state); + init_8(&keys, ivs, &state, 128, 0, NULL, use_gfni); /* Generate 32 bytes at a time */ - asm_ZucGenKeystream32B_8_avx2(&state, (uint32_t **)pKeyStrArr); + keygen_8(&state, pKeyStrArr, 32, use_gfni); /* Point at the next 32 bytes of the key */ for (i = 0; i < NUM_AVX2_BUFS; i++) @@ -756,11 +803,10 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], numKeyStr++; /* Generate the next key stream 8 bytes or 32 bytes */ if (!remainCommonBits && allCommonBits) - asm_ZucGenKeystream8B_8_avx2(&state, - (uint32_t **)pKeyStrArr); + keygen_8(&state, pKeyStrArr, 8, use_gfni); else - asm_ZucGenKeystream32B_8_avx2(&state, - (uint32_t **)pKeyStrArr); + keygen_8(&state, pKeyStrArr, 32, use_gfni); + for (i = 0; i < NUM_AVX2_BUFS; i++) { if (job_in_lane[i] == NULL) continue; @@ -843,13 +889,37 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], #endif } -void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], +void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], + const uint8_t *pIv, + const void * const pBufferIn[NUM_AVX2_BUFS], + uint32_t *pMacI[NUM_AVX2_BUFS], + const uint16_t lengthInBits[NUM_AVX2_BUFS], + const void * const job_in_lane[NUM_AVX2_BUFS]) +{ + _zuc_eia3_8_buffer_job(pKey, pIv, pBufferIn, pMacI, lengthInBits, + job_in_lane, 0); +} + +void zuc_eia3_8_buffer_job_gfni_avx2(const void * const pKey[NUM_AVX2_BUFS], + const uint8_t *pIv, + const void * const pBufferIn[NUM_AVX2_BUFS], + uint32_t *pMacI[NUM_AVX2_BUFS], + const uint16_t lengthInBits[NUM_AVX2_BUFS], + const void * const job_in_lane[NUM_AVX2_BUFS]) +{ + _zuc_eia3_8_buffer_job(pKey, pIv, pBufferIn, pMacI, lengthInBits, + job_in_lane, 1); +} + +static inline +void _zuc256_eia3_8_buffer_job(const void * const pKey[NUM_AVX2_BUFS], const uint8_t *ivs, const void * const pBufferIn[NUM_AVX2_BUFS], void *pMacI[NUM_AVX2_BUFS], const uint16_t lengthInBits[NUM_AVX2_BUFS], const void * const job_in_lane[NUM_AVX2_BUFS], - const uint64_t tag_size) + const uint64_t tag_size, + const unsigned use_gfni) { unsigned int i = 0; DECLARE_ALIGNED(ZucState8_t state, 64); @@ -873,10 +943,10 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], keys.pKeys[i] = pKey[i]; } - asm_Zuc256Initialization_8_avx2(&keys, ivs, &state, T, tag_size); + init_8(&keys, ivs, &state, 256, tag_size, T, use_gfni); /* Generate 32 bytes at a time */ - asm_ZucGenKeystream32B_8_avx2(&state, (uint32_t **)pKeyStrArr); + keygen_8(&state, pKeyStrArr, 32, use_gfni); /* Point at the next 32 bytes of the key */ for (i = 0; i < NUM_AVX2_BUFS; i++) @@ -886,19 +956,10 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], remainCommonBits -= keyStreamLengthInBits; numKeyStr++; /* Generate the next key stream 4/8/16 bytes or 32 bytes */ - if (!remainCommonBits && allCommonBits) { - if (tag_size == 4) - asm_ZucGenKeystream4B_8_avx2(&state, - pKeyStrArr); - else if (tag_size == 8) - asm_ZucGenKeystream8B_8_avx2(&state, - pKeyStrArr); - else - asm_ZucGenKeystream16B_8_avx2(&state, - pKeyStrArr); - } else - asm_ZucGenKeystream32B_8_avx2(&state, - (uint32_t **)pKeyStrArr); + if (!remainCommonBits && allCommonBits) + keygen_8(&state, pKeyStrArr, tag_size, use_gfni); + else + keygen_8(&state, (uint32_t **)pKeyStrArr, 32, use_gfni); for (i = 0; i < NUM_AVX2_BUFS; i++) { void *tag = (void *) &T[i*tag_size]; @@ -991,6 +1052,30 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], #endif } +void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], + const uint8_t *pIv, + const void * const pBufferIn[NUM_AVX2_BUFS], + void *pMacI[NUM_AVX2_BUFS], + const uint16_t lengthInBits[NUM_AVX2_BUFS], + const void * const job_in_lane[NUM_AVX2_BUFS], + const uint64_t tag_size) +{ + _zuc256_eia3_8_buffer_job(pKey, pIv, pBufferIn, pMacI, lengthInBits, + job_in_lane, tag_size, 0); +} + +void zuc256_eia3_8_buffer_job_gfni_avx2(const void * const pKey[NUM_AVX2_BUFS], + const uint8_t *pIv, + const void * const pBufferIn[NUM_AVX2_BUFS], + void *pMacI[NUM_AVX2_BUFS], + const uint16_t lengthInBits[NUM_AVX2_BUFS], + const void * const job_in_lane[NUM_AVX2_BUFS], + const uint64_t tag_size) +{ + _zuc256_eia3_8_buffer_job(pKey, pIv, pBufferIn, pMacI, lengthInBits, + job_in_lane, tag_size, 1); +} + void zuc_eia3_n_buffer_avx2(const void * const pKey[], const void * const pIv[], const void * const pBufferIn[], diff --git a/lib/avx2_t1/zuc_x8_avx2.asm b/lib/avx2_t1/zuc_x8_avx2.asm index 20877dd04ca5d366a92a22316e72261234dc28c5..5c32833e61a223730e498a1a4f43e53737c33c9f 100644 --- a/lib/avx2_t1/zuc_x8_avx2.asm +++ b/lib/avx2_t1/zuc_x8_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,25 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/zuc_sbox.inc" -%include "include/transpose_avx2.asm" -%include "include/memcpy.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/transpose_avx2.inc" +%include "include/memcpy.inc" +%include "include/mb_mgr_datastruct.inc" %include "include/cet.inc" +%ifndef ZUC_CIPHER_8 +%define ZUC_CIPHER_8 asm_ZucCipher_8_avx2 +%define ZUC128_INIT_8 asm_ZucInitialization_8_avx2 +%define ZUC256_INIT_8 asm_Zuc256Initialization_8_avx2 +%define ZUC_KEYGEN32B_8 asm_ZucGenKeystream32B_8_avx2 +%define ZUC_KEYGEN16B_8 asm_ZucGenKeystream16B_8_avx2 +%define ZUC_KEYGEN8B_8 asm_ZucGenKeystream8B_8_avx2 +%define ZUC_KEYGEN4B_8 asm_ZucGenKeystream4B_8_avx2 +%define USE_GFNI 0 +%endif + %ifdef LINUX %define arg1 rdi %define arg2 rsi @@ -437,8 +448,8 @@ align 64 vshufpd %%YTMP4, %%YTMP2, %%YTMP1, 0xFF ; All S1 input values ; Compute S0 and S1 values - S0_comput_AVX2 %%YTMP3, %%YTMP1, %%YTMP2 - S1_comput_AVX2 %%YTMP4, %%YTMP1, %%YTMP2, %%YTMP5 + S0_comput_AVX2 %%YTMP3, %%YTMP1, %%YTMP2, USE_GFNI + S1_comput_AVX2 %%YTMP4, %%YTMP1, %%YTMP2, %%YTMP5, USE_GFNI ; Need to shuffle back %%YTMP1 & %%YTMP2 before storing output ; (revert what was done before S0 and S1 computations) @@ -978,15 +989,15 @@ align 64 FUNC_RESTORE %endmacro -MKGLOBAL(asm_ZucInitialization_8_avx2,function,internal) -asm_ZucInitialization_8_avx2: +MKGLOBAL(ZUC128_INIT_8,function,internal) +ZUC128_INIT_8: endbranch64 ZUC_INIT_8 128, 0 ret -MKGLOBAL(asm_Zuc256Initialization_8_avx2,function,internal) -asm_Zuc256Initialization_8_avx2: +MKGLOBAL(ZUC256_INIT_8,function,internal) +ZUC256_INIT_8: %define tags arg4 %define tag_sz arg5 @@ -1142,8 +1153,8 @@ init_for_cipher: ;; RDI - pSta ;; RSI - pKeyStr ;; -MKGLOBAL(asm_ZucGenKeystream32B_8_avx2,function,internal) -asm_ZucGenKeystream32B_8_avx2: +MKGLOBAL(ZUC_KEYGEN32B_8,function,internal) +ZUC_KEYGEN32B_8: endbranch64 KEYGEN_8_AVX2 8 vzeroupper @@ -1160,8 +1171,8 @@ asm_ZucGenKeystream32B_8_avx2: ;; RDI - pSta ;; RSI - pKeyStr ;; -MKGLOBAL(asm_ZucGenKeystream16B_8_avx2,function,internal) -asm_ZucGenKeystream16B_8_avx2: +MKGLOBAL(ZUC_KEYGEN16B_8,function,internal) +ZUC_KEYGEN16B_8: endbranch64 KEYGEN_8_AVX2 4 vzeroupper @@ -1178,8 +1189,8 @@ asm_ZucGenKeystream16B_8_avx2: ;; RDI - pSta ;; RSI - pKeyStr ;; -MKGLOBAL(asm_ZucGenKeystream8B_8_avx2,function,internal) -asm_ZucGenKeystream8B_8_avx2: +MKGLOBAL(ZUC_KEYGEN8B_8,function,internal) +ZUC_KEYGEN8B_8: endbranch64 KEYGEN_8_AVX2 2 vzeroupper @@ -1196,8 +1207,8 @@ asm_ZucGenKeystream8B_8_avx2: ;; RDI - pSta ;; RSI - pKeyStr ;; -MKGLOBAL(asm_ZucGenKeystream4B_8_avx2,function,internal) -asm_ZucGenKeystream4B_8_avx2: +MKGLOBAL(ZUC_KEYGEN4B_8,function,internal) +ZUC_KEYGEN4B_8: endbranch64 KEYGEN_8_AVX2 1 vzeroupper @@ -1404,8 +1415,8 @@ asm_ZucGenKeystream4B_8_avx2: ;; RCX - lengths ;; R8 - min_length ;; -MKGLOBAL(asm_ZucCipher_8_avx2,function,internal) -asm_ZucCipher_8_avx2: +MKGLOBAL(ZUC_CIPHER_8,function,internal) +ZUC_CIPHER_8: %define pState arg1 %define pIn arg2 %define pOut arg3 diff --git a/lib/avx2_t2/aes128_ecb_vaes_avx2.asm b/lib/avx2_t2/aes128_ecb_vaes_avx2.asm index e0c8a9deb35f2a85990047535bed546c755305b6..941fe273aab79c2414b2c66a8cce48e252d72fcc 100644 --- a/lib/avx2_t2/aes128_ecb_vaes_avx2.asm +++ b/lib/avx2_t2/aes128_ecb_vaes_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -42,9 +42,9 @@ ; arg 4: LEN: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" -%include "include/aes_common.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" %ifdef LINUX %define IN rdi diff --git a/lib/avx2_t2/aes192_ecb_vaes_avx2.asm b/lib/avx2_t2/aes192_ecb_vaes_avx2.asm index d400c3b3af619595a6567dba328bdd65b9979153..c78eeafe605b32da2f6efdc25099e15cb3cbfd9d 100644 --- a/lib/avx2_t2/aes192_ecb_vaes_avx2.asm +++ b/lib/avx2_t2/aes192_ecb_vaes_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,5 +29,5 @@ %define AES_ECB_NROUNDS 12 -%include "include/os.asm" +%include "include/os.inc" %include "avx2_t2/aes128_ecb_vaes_avx2.asm" diff --git a/lib/avx2_t2/aes256_ecb_vaes_avx2.asm b/lib/avx2_t2/aes256_ecb_vaes_avx2.asm index 8c8a730802b0aa79aa56c96c832ae11eba72cf01..e9f14450f38c95ba25000c1826a43d77f10eebb5 100644 --- a/lib/avx2_t2/aes256_ecb_vaes_avx2.asm +++ b/lib/avx2_t2/aes256_ecb_vaes_avx2.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,5 +29,5 @@ %define AES_ECB_NROUNDS 14 -%include "include/os.asm" +%include "include/os.inc" %include "avx2_t2/aes128_ecb_vaes_avx2.asm" diff --git a/lib/avx2_t2/mb_mgr_avx2_t2.c b/lib/avx2_t2/mb_mgr_avx2_t2.c index 55f33beb8b4e3877f6cb078a3465151c3a572a10..63cd1325bfa2ed9425ad40a4c9a4fb51152189a9 100644 --- a/lib/avx2_t2/mb_mgr_avx2_t2.c +++ b/lib/avx2_t2/mb_mgr_avx2_t2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -73,6 +73,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2_t2 #define SUBMIT_HASH_BURST submit_hash_burst_avx2_t2 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx2_t2 +#define SET_SUITE_ID_FN set_suite_id_avx2_t2 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX2 @@ -205,14 +206,14 @@ #define POLY1305_MAC poly1305_mac_scalar /* ZUC EEA3 & EIA3 */ -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx2 -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx2 -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx2 -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx2 -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx2 -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx2 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx2 -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx2 +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_gfni_avx2 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_gfni_avx2 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_gfni_avx2 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_gfni_avx2 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_gfni_avx2 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_gfni_avx2 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_gfni_avx2 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_gfni_avx2 /* SNOW-V */ #define SUBMIT_JOB_SNOW_V snow_v_avx @@ -345,7 +346,6 @@ init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) state->earliest_job = -1; } - /* set handlers */ /* set handlers */ state->get_next_job = GET_NEXT_JOB; state->submit_job = SUBMIT_JOB; @@ -361,6 +361,7 @@ init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_avx2; state->keyexp_192 = aes_keyexp_192_avx2; @@ -372,12 +373,12 @@ init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) state->xcbc_keyexp = aes_xcbc_expand_key_avx2; state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx2; - state->sha1 = sha1_avx2; - state->sha224_one_block = sha224_one_block_avx2; - state->sha224 = sha224_avx2; - state->sha256_one_block = sha256_one_block_avx2; - state->sha256 = sha256_avx2; + state->sha1_one_block = sha1_one_block_sse_shani; + state->sha1 = sha1_sse_shani; + state->sha224_one_block = sha224_one_block_sse_shani; + state->sha224 = sha224_sse_shani; + state->sha256_one_block = sha256_one_block_sse_shani; + state->sha256 = sha256_sse_shani; state->sha384_one_block = sha384_one_block_avx2; state->sha384 = sha384_avx2; state->sha512_one_block = sha512_one_block_avx2; @@ -385,6 +386,7 @@ init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_avx2; state->aes128_cfb_one = aes_cfb_128_one_avx2; + state->aes256_cfb_one = aes_cfb_256_one_avx2; state->eea3_1_buffer = zuc_eea3_1_buffer_avx2; state->eea3_4_buffer = zuc_eea3_4_buffer_avx; @@ -469,7 +471,7 @@ init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) state->gcm256_pre = aes_gcm_pre_256_avx_gen4; state->ghash = ghash_avx_gen4; - state->ghash_pre = ghash_pre_avx_gen2; + state->ghash_pre = ghash_pre_avx_gen4; state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; @@ -480,6 +482,9 @@ init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_avx; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_avx; } #include "mb_mgr_code.h" diff --git a/lib/avx2_t2/mb_mgr_zuc_submit_flush_gfni_avx2.asm b/lib/avx2_t2/mb_mgr_zuc_submit_flush_gfni_avx2.asm new file mode 100644 index 0000000000000000000000000000000000000000..88859b3d6f6af876df3c0800e86cdaa71c0d41f4 --- /dev/null +++ b/lib/avx2_t2/mb_mgr_zuc_submit_flush_gfni_avx2.asm @@ -0,0 +1,40 @@ +;; +;; Copyright (c) 2022-2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; +%define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_gfni_avx2 +%define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_gfni_avx2 +%define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_gfni_avx2 +%define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_gfni_avx2 +%define SUBMIT_JOB_ZUC128_EIA3 submit_job_zuc_eia3_gfni_avx2 +%define FLUSH_JOB_ZUC128_EIA3 flush_job_zuc_eia3_gfni_avx2 +%define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_gfni_avx2 +%define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_gfni_avx2 +%define ZUC128_INIT_8 asm_ZucInitialization_8_gfni_avx2 +%define ZUC256_INIT_8 asm_Zuc256Initialization_8_gfni_avx2 +%define ZUC_EIA3_8_BUFFER zuc_eia3_8_buffer_job_gfni_avx2 +%define ZUC256_EIA3_8_BUFFER zuc256_eia3_8_buffer_job_gfni_avx2 +%define ZUC_CIPHER_8 asm_ZucCipher_8_gfni_avx2 +%include "avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm" diff --git a/lib/avx2_t2/zuc_x8_gfni_avx2.asm b/lib/avx2_t2/zuc_x8_gfni_avx2.asm new file mode 100644 index 0000000000000000000000000000000000000000..44159fe31cc348b6e76178d530ed2e9f4ee8695d --- /dev/null +++ b/lib/avx2_t2/zuc_x8_gfni_avx2.asm @@ -0,0 +1,37 @@ +;; +;; Copyright (c) 2022-2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%define USE_GFNI 1 +%define ZUC_CIPHER_8 asm_ZucCipher_8_gfni_avx2 +%define ZUC128_INIT_8 asm_ZucInitialization_8_gfni_avx2 +%define ZUC256_INIT_8 asm_Zuc256Initialization_8_gfni_avx2 +%define ZUC_KEYGEN32B_8 asm_ZucGenKeystream32B_8_gfni_avx2 +%define ZUC_KEYGEN16B_8 asm_ZucGenKeystream16B_8_gfni_avx2 +%define ZUC_KEYGEN8B_8 asm_ZucGenKeystream8B_8_gfni_avx2 +%define ZUC_KEYGEN4B_8 asm_ZucGenKeystream4B_8_gfni_avx2 +%include "avx2_t1/zuc_x8_avx2.asm" + diff --git a/lib/avx2_t3/README b/lib/avx2_t3/README new file mode 100644 index 0000000000000000000000000000000000000000..1d77c4f1188177fdb571d589899e1198c8379e0c --- /dev/null +++ b/lib/avx2_t3/README @@ -0,0 +1,3 @@ +AVX2 TYPE3: +- AVX2 TYPE2: AVX2, BMI2, AESNI, PCLMULQDQ, CMOV, VAES, VPCLMULQDQ, SHANI, GFNI +- AVXIFMA diff --git a/lib/avx2_t3/mb_mgr_avx2_t3.c b/lib/avx2_t3/mb_mgr_avx2_t3.c new file mode 100644 index 0000000000000000000000000000000000000000..d17a62410e14840e18dc83863b3a84ec3780d78a --- /dev/null +++ b/lib/avx2_t3/mb_mgr_avx2_t3.c @@ -0,0 +1,497 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define AVX2 + +#include "ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/snow3g_submit.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_sse_type1.h" /* poly1305, snow3g */ +#include "include/arch_sse_type2.h" /* shani */ +#include "include/arch_avx_type1.h" +#include "include/arch_avx2_type1.h" +#include "include/arch_avx2_type2.h" +#include "include/arch_avx2_type3.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +/* JOB API */ +#define SUBMIT_JOB submit_job_avx2_t3 +#define FLUSH_JOB flush_job_avx2_t3 +#define QUEUE_SIZE queue_size_avx2_t3 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx2_t3 +#define GET_NEXT_JOB get_next_job_avx2_t3 +#define GET_COMPLETED_JOB get_completed_job_avx2_t3 +#define GET_NEXT_BURST get_next_burst_avx2_t3 +#define SUBMIT_BURST submit_burst_avx2_t3 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2_t3 +#define FLUSH_BURST flush_burst_avx2_t3 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx2_t3 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2_t3 +#define SUBMIT_HASH_BURST submit_hash_burst_avx2_t3 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx2_t3 +#define SET_SUITE_ID_FN set_suite_id_avx2_t3 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX2 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX2 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX2 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX2 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX2 + +/* AES-GCM */ +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen4 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen4 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen4 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen4 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen4 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen4 + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_gcm_dec_avx2 +#define SUBMIT_JOB_AES_GCM_ENC submit_job_gcm_enc_avx2 + +/* AES-CBC */ +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx + +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx + +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_vaes_avx2 + +#define AES_ECB_ENC_128 aes_ecb_enc_128_vaes_avx2 +#define AES_ECB_ENC_192 aes_ecb_enc_192_vaes_avx2 +#define AES_ECB_ENC_256 aes_ecb_enc_256_vaes_avx2 +#define AES_ECB_DEC_128 aes_ecb_dec_128_vaes_avx2 +#define AES_ECB_DEC_192 aes_ecb_dec_192_vaes_avx2 +#define AES_ECB_DEC_256 aes_ecb_dec_256_vaes_avx2 + +/* AES-CTR */ +#define AES_CTR_128 aes_cntr_128_avx +#define AES_CTR_192 aes_cntr_192_avx +#define AES_CTR_256 aes_cntr_256_avx +#define AES_CTR_128_BIT aes_cntr_bit_128_avx +#define AES_CTR_192_BIT aes_cntr_bit_192_avx +#define AES_CTR_256_BIT aes_cntr_bit_256_avx + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_avx2 +#define AES_CFB_256_ONE aes_cfb_256_one_avx2 + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +/* SHA1/224/256/384/512 */ +/* note: SHA1 MB is better than SHANI on Xeon processors */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 +#define FLUSH_JOB_SHA1 flush_job_sha1_avx2 +#define SUBMIT_JOB_SHA224 submit_job_sha224_ni_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_ni_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_ni_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx2 +#define FLUSH_JOB_SHA384 flush_job_sha384_avx2 +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 +#define FLUSH_JOB_SHA512 flush_job_sha512_avx2 + +/* HMAC-SHA1/224/256/384/512 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_avx2 +#define FLUSH_JOB_HMAC flush_job_hmac_avx2 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_ni_sse +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_ni_sse +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_ni_sse +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_ni_sse +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx2 +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx2 +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx2 +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx2 +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx2 +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx2 +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx2 +#define POLY1305_MAC poly1305_mac_fma_avx2 + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_gfni_avx2 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_gfni_avx2 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_gfni_avx2 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_gfni_avx2 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_gfni_avx2 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_gfni_avx2 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_gfni_avx2 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_gfni_avx2 + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_avx +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx + +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_avx2_t2(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_avx2_t2(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx2_t2 +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx2_t2 + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_avx_local + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES CBC. + */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 8); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, 2); + + /* Init HMAC/SHA_256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, 2); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX2_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX2_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); + + /* Init AES CBC-S out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX2_NUM_SHA1_LANES); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, 2); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, 2); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX2_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); +} + +IMB_DLL_LOCAL void +init_mb_mgr_avx2_t3_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for AVX2 interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX2) != IMB_CPUFLAGS_AVX2) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_AVX2; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; + + state->keyexp_128 = aes_keyexp_128_avx2; + state->keyexp_192 = aes_keyexp_192_avx2; + state->keyexp_256 = aes_keyexp_256_avx2; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx2; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx2; + + state->xcbc_keyexp = aes_xcbc_expand_key_avx2; + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_sse_shani; + state->sha1 = sha1_sse_shani; + state->sha224_one_block = sha224_one_block_sse_shani; + state->sha224 = sha224_sse_shani; + state->sha256_one_block = sha256_one_block_sse_shani; + state->sha256 = sha256_sse_shani; + state->sha384_one_block = sha384_one_block_avx2; + state->sha384 = sha384_avx2; + state->sha512_one_block = sha512_one_block_avx2; + state->sha512 = sha512_avx2; + state->md5_one_block = md5_one_block_avx2; + + state->aes128_cfb_one = aes_cfb_128_one_avx2; + state->aes256_cfb_one = aes_cfb_256_one_avx2; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx2; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eea3_n_buffer = zuc_eea3_n_buffer_avx2; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx2; + state->eia3_n_buffer = zuc_eia3_n_buffer_avx2; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx2; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx2; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx2; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx2; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx2; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx2; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx2; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx2; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx2; + state->snow3g_init_key_sched = snow3g_init_key_sched_avx2; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx2; + + state->hec_32 = hec_32_avx; + state->hec_64 = hec_64_avx; + + state->crc32_ethernet_fcs = ethernet_fcs_avx; + state->crc16_x25 = crc16_x25_avx; + state->crc32_sctp = crc32_sctp_avx; + state->crc24_lte_a = crc24_lte_a_avx; + state->crc24_lte_b = crc24_lte_b_avx; + state->crc16_fp_data = crc16_fp_data_avx; + state->crc11_fp_header = crc11_fp_header_avx; + state->crc7_fp_header = crc7_fp_header_avx; + state->crc10_iuup_data = crc10_iuup_data_avx; + state->crc6_iuup_header = crc6_iuup_header_avx; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; + +#ifdef AVX_IFMA + state->chacha20_poly1305_init = + init_chacha20_poly1305_fma_avx2; + state->chacha20_poly1305_enc_update = + update_enc_chacha20_poly1305_fma_avx2; + state->chacha20_poly1305_dec_update = + update_dec_chacha20_poly1305_fma_avx2; + state->chacha20_poly1305_finalize = + finalize_chacha20_poly1305_fma_avx2; +#endif + + state->gcm128_enc = aes_gcm_enc_128_avx_gen4; + state->gcm192_enc = aes_gcm_enc_192_avx_gen4; + state->gcm256_enc = aes_gcm_enc_256_avx_gen4; + state->gcm128_dec = aes_gcm_dec_128_avx_gen4; + state->gcm192_dec = aes_gcm_dec_192_avx_gen4; + state->gcm256_dec = aes_gcm_dec_256_avx_gen4; + state->gcm128_init = aes_gcm_init_128_avx_gen4; + state->gcm192_init = aes_gcm_init_192_avx_gen4; + state->gcm256_init = aes_gcm_init_256_avx_gen4; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen4; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen4; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen4; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen4; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen4; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen4; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen4; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen4; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen4; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen4; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen4; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen4; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen4; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen4; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen4; + state->gcm128_precomp = aes_gcm_precomp_128_avx_gen4; + state->gcm192_precomp = aes_gcm_precomp_192_avx_gen4; + state->gcm256_precomp = aes_gcm_precomp_256_avx_gen4; + state->gcm128_pre = aes_gcm_pre_128_avx_gen4; + state->gcm192_pre = aes_gcm_pre_192_avx_gen4; + state->gcm256_pre = aes_gcm_pre_256_avx_gen4; + + state->ghash = ghash_avx_gen4; + state->ghash_pre = ghash_pre_avx_gen4; + + state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; + state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; + state->gmac256_init = imb_aes_gmac_init_256_avx_gen4; + state->gmac128_update = imb_aes_gmac_update_128_avx_gen4; + state->gmac192_update = imb_aes_gmac_update_192_avx_gen4; + state->gmac256_update = imb_aes_gmac_update_256_avx_gen4; + state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; + state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; + state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_avx; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_avx; +} + +#include "mb_mgr_code.h" diff --git a/lib/avx2_t3/poly_fma_avx2.asm b/lib/avx2_t3/poly_fma_avx2.asm new file mode 100644 index 0000000000000000000000000000000000000000..cc9f84f253c295095c2f3eb6640e83f0ec44084f --- /dev/null +++ b/lib/avx2_t3/poly_fma_avx2.asm @@ -0,0 +1,1145 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" +%include "include/cet.inc" + +;; Enforce VEX encoding for AVX2 capable systems +%xdefine vpmadd52luq {vex3}vpmadd52luq +%xdefine vpmadd52huq {vex3}vpmadd52huq + +[bits 64] +default rel + +align 32 +mask_44: +dq 0xfffffffffff, 0xfffffffffff, 0xfffffffffff, 0xfffffffffff + +align 32 +mask_42: +dq 0x3ffffffffff, 0x3ffffffffff, 0x3ffffffffff, 0x3ffffffffff + +align 32 +high_bit: +dq 0x10000000000, 0x10000000000, 0x10000000000, 0x10000000000 + +align 16 +pad16_bit: +dq 0x01, 0x0 +dq 0x0100, 0x0 +dq 0x010000, 0x0 +dq 0x01000000, 0x0 +dq 0x0100000000, 0x0 +dq 0x010000000000, 0x0 +dq 0x01000000000000, 0x0 +dq 0x0100000000000000, 0x0 +dq 0x0, 0x01 +dq 0x0, 0x0100 +dq 0x0, 0x010000 +dq 0x0, 0x01000000 +dq 0x0, 0x0100000000 +dq 0x0, 0x010000000000 +dq 0x0, 0x01000000000000 +dq 0x0, 0x0100000000000000 + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx + +%define job arg1 +%define gp1 rsi +%define gp2 rcx + +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 + +%define job rdi +%define gp1 rcx ;; 'arg1' copied to 'job' at start +%define gp2 rsi +%endif + +;; don't use rdx and rax - they are needed for multiply operation +%define gp3 rbp +%define gp4 r8 +%define gp5 r9 +%define gp6 r10 +%define gp7 r11 +%define gp8 r12 +%define gp9 r13 +%define gp10 r14 +%define gp11 r15 + +%xdefine len gp11 +%xdefine msg gp10 + +%define POLY1305_BLOCK_SIZE 16 + +%define APPEND(a,b) a %+ b + +struc STACKFRAME +_r4_r1_save: resy 3 ; Memory to save limbs of powers of R +_r4_save: resy 3 ; Memory to save limbs of powers of R +_r4p_save: resy 2 ; Memory to save limbs of powers of R +_gpr_save: resq 8 ; Memory to save GP registers +_xmm_save: reso 10 ; Memory to save XMM registers +_rsp_save: resq 1 ; Memory to save RSP +endstruc + +mksection .text + +;; ============================================================================= +;; ============================================================================= +;; Initializes POLY1305 context structure +;; ============================================================================= +%macro POLY1305_INIT 6 +%define %%KEY %1 ; [in] pointer to 32-byte key +%define %%A0 %2 ; [out] GPR with accumulator bits 63..0 +%define %%A1 %3 ; [out] GPR with accumulator bits 127..64 +%define %%A2 %4 ; [out] GPR with accumulator bits 195..128 +%define %%R0 %5 ; [out] GPR with R constant bits 63..0 +%define %%R1 %6 ; [out] GPR with R constant bits 127..64 + + ;; R = KEY[0..15] & 0xffffffc0ffffffc0ffffffc0fffffff + mov %%R0, 0x0ffffffc0fffffff + and %%R0, [%%KEY + (0 * 8)] + + mov %%R1, 0x0ffffffc0ffffffc + and %%R1, [%%KEY + (1 * 8)] + + ;; set accumulator to 0 + xor %%A0, %%A0 + xor %%A1, %%A1 + xor %%A2, %%A2 +%endmacro + +;; ============================================================================= +;; ============================================================================= +;; Computes hash for message length being multiple of block size +;; ============================================================================= +%macro POLY1305_MUL_REDUCE 11-12 +%define %%A0 %1 ; [in/out] GPR with accumulator bits 63:0 +%define %%A1 %2 ; [in/out] GPR with accumulator bits 127:64 +%define %%A2 %3 ; [in/out] GPR with accumulator bits 195:128 +%define %%R0 %4 ; [in] GPR with R constant bits 63:0 +%define %%R1 %5 ; [in] GPR with R constant bits 127:64 +%define %%C1 %6 ; [in] C1 = R1 + (R1 >> 2) +%define %%T1 %7 ; [clobbered] GPR register +%define %%T2 %8 ; [clobbered] GPR register +%define %%T3 %9 ; [clobbered] GPR register +%define %%GP_RAX %10 ; [clobbered] RAX register +%define %%GP_RDX %11 ; [clobbered] RDX register +%define %%ONLY128 %12 ; [in] Used if input A2 is 0 + + ;; Combining 64-bit x 64-bit multiplication with reduction steps + ;; + ;; NOTES: + ;; 1) A2 here is only two bits so anything above is subject of reduction. + ;; Constant C1 = R1 + (R1 >> 2) simplifies multiply with less operations + ;; 2) Magic 5x comes from mod 2^130-5 property and incorporating + ;; reduction into multiply phase. + ;; See "Cheating at modular arithmetic" and "Poly1305's prime: 2^130 - 5" + ;; paragraphs at https://loup-vaillant.fr/tutorials/poly1305-design for more details. + ;; + ;; Flow of the code below is as follows: + ;; + ;; A2 A1 A0 + ;; x R1 R0 + ;; ----------------------------- + ;; A2×R0 A1×R0 A0×R0 + ;; + A0×R1 + ;; + 5xA2xR1 5xA1xR1 + ;; ----------------------------- + ;; [0|L2L] [L1H|L1L] [L0H|L0L] + ;; + ;; Registers: T3:T2 T1:A0 + ;; + ;; Completing the multiply and adding (with carry) 3x128-bit limbs into + ;; 192-bits again (3x64-bits): + ;; A0 = L0L + ;; A1 = L0H + L1L + ;; T3 = L1H + L2L + + ;; T3:T2 = (A0 * R1) + mov %%GP_RAX, %%R1 + mul %%A0 + mov %%T2, %%GP_RAX + mov %%GP_RAX, %%R0 + mov %%T3, %%GP_RDX + + ;; T1:A0 = (A0 * R0) + mul %%A0 + mov %%A0, %%GP_RAX ;; A0 not used in other operations + mov %%GP_RAX, %%R0 + mov %%T1, %%GP_RDX + + ;; T3:T2 += (A1 * R0) + mul %%A1 + add %%T2, %%GP_RAX + mov %%GP_RAX, %%C1 + adc %%T3, %%GP_RDX + + ;; T1:A0 += (A1 * R1x5) + mul %%A1 +%if %0 == 11 + mov %%A1, %%A2 ;; use A1 for A2 +%endif + add %%A0, %%GP_RAX + adc %%T1, %%GP_RDX + + ;; NOTE: A2 is clamped to 2-bits, + ;; R1/R0 is clamped to 60-bits, + ;; their product is less than 2^64. + +%if %0 == 11 + ;; T3:T2 += (A2 * R1x5) + imul %%A1, %%C1 + add %%T2, %%A1 + mov %%A1, %%T1 ;; T1:A0 => A1:A0 + adc %%T3, 0 + + ;; T3:A1 += (A2 * R0) + imul %%A2, %%R0 + add %%A1, %%T2 + adc %%T3, %%A2 + ;; If A2 == 0, just move and add T1-T2 to A1 +%else + mov %%A1, %%T1 + add %%A1, %%T2 + adc %%T3, 0 +%endif + + ;; At this point, 3 64-bit limbs are in T3:A1:A0 + ;; T3 can span over more than 2 bits so final partial reduction step is needed. + ;; + ;; Partial reduction (just to fit into 130 bits) + ;; A2 = T3 & 3 + ;; k = (T3 & ~3) + (T3 >> 2) + ;; Y x4 + Y x1 + ;; A2:A1:A0 += k + ;; + ;; Result will be in A2:A1:A0 + mov %%T1, %%T3 + mov DWORD(%%A2), DWORD(%%T3) + and %%T1, ~3 + shr %%T3, 2 + and DWORD(%%A2), 3 + add %%T1, %%T3 + + ;; A2:A1:A0 += k (kept in T1) + add %%A0, %%T1 + adc %%A1, 0 + adc DWORD(%%A2), 0 +%endmacro + +;; ============================================================================= +;; ============================================================================= +;; Computes hash for 4 16-byte message blocks, +;; and adds new message blocks to accumulator, +;; interleaving this computation with the loading and splatting +;; of new data. +;; +;; It first multiplies all 4 blocks with powers of R +;; +;; a2 a1 a0 +;; × b2 b1 b0 +;; --------------------------------------- +;; a2×b0 a1×b0 a0×b0 +;; + a1×b1 a0×b1 5×a2×b1 +;; + a0×b2 5×a2×b2 5×a1×b2 +;; --------------------------------------- +;; p2 p1 p0 +;; +;; Then, it propagates the carry (higher bits after bit 43) +;; from lower limbs into higher limbs, +;; multiplying by 5 in case of the carry of p2, and adds +;; the results to A0-A2 and B0-B2. +;; +;; ============================================================================= +%macro POLY1305_MSG_MUL_REDUCE_VEC4 22 +%define %%A0 %1 ; [in/out] YMM register containing 1st 44-bit limb of blocks 1-4 +%define %%A1 %2 ; [in/out] YMM register containing 2nd 44-bit limb of blocks 1-4 +%define %%A2 %3 ; [in/out] YMM register containing 3rd 44-bit limb of blocks 1-4 +%define %%R0 %4 ; [in] YMM register/memory (R0) to include the 1st limb of R +%define %%R1 %5 ; [in] YMM register/memory (R1) to include the 2nd limb of R +%define %%R2 %6 ; [in] YMM register/memory (R2) to include the 3rd limb of R +%define %%R1P %7 ; [in] YMM register/memory (R1') to include the 2nd limb of R (multiplied by 5) +%define %%R2P %8 ; [in] YMM register/memory (R2') to include the 3rd limb of R (multiplied by 5) +%define %%P0_L %9 ; [clobbered] YMM register to contain p[0] of the 4 blocks 1-4 +%define %%P0_H %10 ; [clobbered] YMM register to contain p[0] of the 4 blocks 1-4 +%define %%P1_L %11 ; [clobbered] YMM register to contain p[1] of the 4 blocks 1-4 +%define %%P1_H %12 ; [clobbered] YMM register to contain p[1] of the 4 blocks 1-4 +%define %%P2_L %13 ; [clobbered] YMM register to contain p[2] of the 4 blocks 1-4 +%define %%P2_H %14 ; [clobbered] YMM register to contain p[2] of the 4 blocks 1-4 +%define %%YTMP1 %15 ; [clobbered] Temporary YMM register +%define %%YTMP2 %16 ; [clobbered] Temporary YMM register +%define %%YTMP3 %17 ; [clobbered] Temporary YMM register +%define %%YTMP4 %18 ; [clobbered] Temporary YMM register +%define %%YTMP5 %19 ; [clobbered] Temporary YMM register +%define %%YTMP6 %20 ; [clobbered] Temporary YMM register +%define %%MSG %21 ; [in/out] Pointer to message +%define %%LEN %22 ; [in/out] Length left of message + + ;; Reset accumulator + vpxor %%P0_L, %%P0_L + vpxor %%P0_H, %%P0_H + vpxor %%P1_L, %%P1_L + vpxor %%P1_H, %%P1_H + vpxor %%P2_L, %%P2_L + vpxor %%P2_H, %%P2_H + + ;; This code interleaves hash computation with input loading/splatting + + ; Calculate products + vpmadd52luq %%P0_L, %%A2, %%R1P + vpmadd52huq %%P0_H, %%A2, %%R1P + ;; input loading of new blocks + add %%MSG, POLY1305_BLOCK_SIZE*4 + sub %%LEN, POLY1305_BLOCK_SIZE*4 + + vpmadd52luq %%P1_L, %%A2, %%R2P + vpmadd52huq %%P1_H, %%A2, %%R2P + ; Load next block of data (64 bytes) + vmovdqu %%YTMP1, [%%MSG] + vmovdqu %%YTMP2, [%%MSG + 32] + + ; Interleave new blocks of data + vpunpckhqdq %%YTMP3, %%YTMP1, %%YTMP2 + vpunpcklqdq %%YTMP1, %%YTMP1, %%YTMP2 + + vpmadd52luq %%P0_L, %%A0, %%R0 + vpmadd52huq %%P0_H, %%A0, %%R0 + ; Highest 42-bit limbs of new blocks + vpsrlq %%YTMP6, %%YTMP3, 24 + vpor %%YTMP6, [rel high_bit] ; Add 2^128 to all 4 final qwords of the message + + ; Middle 44-bit limbs of new blocks + vpsrlq %%YTMP2, %%YTMP1, 44 + vpsllq %%YTMP4, %%YTMP3, 20 + + vpmadd52luq %%P2_L, %%A2, %%R0 + vpmadd52huq %%P2_H, %%A2, %%R0 + vpor %%YTMP2, %%YTMP4 + vpand %%YTMP2, [rel mask_44] + + ; Lowest 44-bit limbs of new blocks + vpand %%YTMP1, [rel mask_44] + + vpmadd52luq %%P1_L, %%A0, %%R1 + vpmadd52huq %%P1_H, %%A0, %%R1 + + vpmadd52luq %%P0_L, %%A1, %%R2P + vpmadd52huq %%P0_H, %%A1, %%R2P + + vpmadd52luq %%P2_L, %%A0, %%R2 + vpmadd52huq %%P2_H, %%A0, %%R2 + ; Carry propagation (first pass) + vpsrlq %%YTMP5, %%P0_L, 44 + vpsllq %%P0_H, 8 + + vpmadd52luq %%P1_L, %%A1, %%R0 + vpmadd52huq %%P1_H, %%A1, %%R0 + ; Carry propagation (first pass) - continue + vpand %%A0, %%P0_L, [rel mask_44] ; Clear top 20 bits + vpaddq %%P0_H, %%YTMP5 + + vpmadd52luq %%P2_L, %%A1, %%R1 + vpmadd52huq %%P2_H, %%A1, %%R1 + ; Carry propagation (first pass) - continue + vpaddq %%P1_L, %%P0_H + vpsllq %%P1_H, 8 + vpsrlq %%YTMP5, %%P1_L, 44 + vpand %%A1, %%P1_L, [rel mask_44] ; Clear top 20 bits + + vpaddq %%P2_L, %%P1_H ; P2_L += P1_H + P1_L[63:44] + vpaddq %%P2_L, %%YTMP5 + vpand %%A2, %%P2_L, [rel mask_42] ; Clear top 22 bits + vpaddq %%A2, %%YTMP6 ; Add highest bits from new blocks to accumulator + vpsrlq %%YTMP5, %%P2_L, 42 + vpsllq %%P2_H, 10 + vpaddq %%P2_H, %%YTMP5 + + ; Carry propagation (second pass) + ; Multiply by 5 the highest bits (above 130 bits) + vpaddq %%A0, %%P2_H + vpsllq %%P2_H, 2 + vpaddq %%A0, %%P2_H + + vpsrlq %%YTMP5, %%A0, 44 + vpand %%A0, [rel mask_44] + vpaddq %%A0, %%YTMP1 ; Add low 42-bit bits from new blocks to accumulator + vpaddq %%A1, %%YTMP2 ; Add medium 42-bit bits from new blocks to accumulator + vpaddq %%A1, %%YTMP5 +%endmacro + +;; ============================================================================= +;; ============================================================================= +;; Computes hash for 4 16-byte message blocks. +;; +;; It first multiplies all 4 blocks with powers of R (4 blocks from A0-A2 +;; multiplied by R0-R2) +;; +;; +;; a2 a1 a0 +;; × b2 b1 b0 +;; --------------------------------------- +;; a2×b0 a1×b0 a0×b0 +;; + a1×b1 a0×b1 5×a2×b1 +;; + a0×b2 5×a2×b2 5×a1×b2 +;; --------------------------------------- +;; p2 p1 p0 +;; +;; Then, it propagates the carry (higher bits after bit 43) from lower limbs into higher limbs, +;; multiplying by 5 in case of the carry of p2. +;; +;; ============================================================================= +%macro POLY1305_MUL_REDUCE_VEC4 16 +%define %%A0 %1 ; [in/out] YMM register containing 1st 44-bit limb of the 4 blocks +%define %%A1 %2 ; [in/out] YMM register containing 2nd 44-bit limb of the 4 blocks +%define %%A2 %3 ; [in/out] YMM register containing 3rd 44-bit limb of the 4 blocks +%define %%R0 %4 ; [in] YMM register/memory (R0) to include the 1st limb of R +%define %%R1 %5 ; [in] YMM register/memory (R1) to include the 2nd limb of R +%define %%R2 %6 ; [in] YMM register/memory (R2) to include the 3rd limb of R +%define %%R1P %7 ; [in] YMM register/memory (R1') to include the 2nd limb of R (multiplied by 5) +%define %%R2P %8 ; [in] YMM register/memory (R2') to include the 3rd limb of R (multiplied by 5) +%define %%P0_L %9 ; [clobbered] YMM register to contain p[0] of the 4 blocks +%define %%P0_H %10 ; [clobbered] YMM register to contain p[0] of the 4 blocks +%define %%P1_L %11 ; [clobbered] YMM register to contain p[1] of the 4 blocks +%define %%P1_H %12 ; [clobbered] YMM register to contain p[1] of the 4 blocks +%define %%P2_L %13 ; [clobbered] YMM register to contain p[2] of the 4 blocks +%define %%P2_H %14 ; [clobbered] YMM register to contain p[2] of the 4 blocks +%define %%YTMP1 %15 ; [clobbered] Temporary YMM register +%define %%YTMP2 %16 ; [clobbered] Temporary YMM register + + ;; Reset accumulator + vpxor %%P0_L, %%P0_L + vpxor %%P0_H, %%P0_H + vpxor %%P1_L, %%P1_L + vpxor %%P1_H, %%P1_H + vpxor %%P2_L, %%P2_L + vpxor %%P2_H, %%P2_H + + ;; This code interleaves hash computation with input loading/splatting + + ; Calculate products + vpmadd52luq %%P0_L, %%A2, %%R1P + vpmadd52huq %%P0_H, %%A2, %%R1P + + vpmadd52luq %%P1_L, %%A2, %%R2P + vpmadd52huq %%P1_H, %%A2, %%R2P + + vpmadd52luq %%P0_L, %%A0, %%R0 + vpmadd52huq %%P0_H, %%A0, %%R0 + + vpmadd52luq %%P2_L, %%A2, %%R0 + vpmadd52huq %%P2_H, %%A2, %%R0 + + vpmadd52luq %%P1_L, %%A0, %%R1 + vpmadd52huq %%P1_H, %%A0, %%R1 + + vpmadd52luq %%P0_L, %%A1, %%R2P + vpmadd52huq %%P0_H, %%A1, %%R2P + + vpmadd52luq %%P2_L, %%A0, %%R2 + vpmadd52huq %%P2_H, %%A0, %%R2 + + ; Carry propagation (first pass) + vpsrlq %%YTMP1, %%P0_L, 44 + vpsllq %%P0_H, 8 + + vpmadd52luq %%P1_L, %%A1, %%R0 + vpmadd52huq %%P1_H, %%A1, %%R0 + + ; Carry propagation (first pass) - continue + vpand %%A0, %%P0_L, [rel mask_44] ; Clear top 20 bits + vpaddq %%P0_H, %%YTMP1 + + vpmadd52luq %%P2_L, %%A1, %%R1 + vpmadd52huq %%P2_H, %%A1, %%R1 + + ; Carry propagation (first pass) - continue + vpaddq %%P1_L, %%P0_H + vpsllq %%P1_H, 8 + vpsrlq %%YTMP1, %%P1_L, 44 + vpand %%A1, %%P1_L, [rel mask_44] ; Clear top 20 bits + + vpaddq %%P2_L, %%P1_H ; P2_L += P1_H + P1_L[63:44] + vpaddq %%P2_L, %%YTMP1 + vpand %%A2, %%P2_L, [rel mask_42] ; Clear top 22 bits + vpsrlq %%YTMP1, %%P2_L, 42 + vpsllq %%P2_H, 10 + vpaddq %%P2_H, %%YTMP1 + + ; Carry propagation (second pass) + ; Multiply by 5 the highest bits (above 130 bits) + vpaddq %%A0, %%P2_H + vpsllq %%P2_H, 2 + vpaddq %%A0, %%P2_H + + vpsrlq %%YTMP1, %%A0, 44 + vpand %%A0, [rel mask_44] + vpaddq %%A1, %%YTMP1 +%endmacro + +;; ============================================================================= +;; ============================================================================= +;; Computes hash for message length being multiple of block size +;; ============================================================================= +%macro POLY1305_BLOCKS 14 +%define %%MSG %1 ; [in/out] GPR pointer to input message (updated) +%define %%LEN %2 ; [in/out] GPR in: length in bytes / out: length mod 16 +%define %%A0 %3 ; [in/out] accumulator bits 63..0 +%define %%A1 %4 ; [in/out] accumulator bits 127..64 +%define %%A2 %5 ; [in/out] accumulator bits 195..128 +%define %%R0 %6 ; [in] R constant bits 63..0 +%define %%R1 %7 ; [in] R constant bits 127..64 +%define %%T0 %8 ; [clobbered] GPR register +%define %%T1 %9 ; [clobbered] GPR register +%define %%T2 %10 ; [clobbered] GPR register +%define %%T3 %11 ; [clobbered] GPR register +%define %%GP_RAX %12 ; [clobbered] RAX register +%define %%GP_RDX %13 ; [clobbered] RDX register +%define %%PAD_16 %14 ; [in] text "pad_to_16" or "no_padding" + +%define %%YMM_ACC0 ymm0 +%define %%YMM_ACC1 ymm1 +%define %%YMM_ACC2 ymm2 + +%define %%YTMP1 ymm3 +%define %%YTMP2 ymm4 +%define %%YTMP3 ymm5 +%define %%YTMP4 ymm6 +%define %%YTMP5 ymm7 +%define %%YTMP6 ymm8 +%define %%YTMP7 ymm9 +%define %%YTMP8 ymm10 +%define %%YTMP9 ymm11 +%define %%YTMP10 ymm12 +%define %%YTMP11 ymm13 +%define %%YTMP12 ymm14 +%define %%YTMP13 ymm15 + +%define %%YMM_R0 %%YTMP11 +%define %%YMM_R1 %%YTMP12 +%define %%YMM_R2 %%YTMP13 + +%define %%XTMP1 XWORD(%%YTMP1) +%define %%XTMP2 XWORD(%%YTMP2) +%define %%XTMP3 XWORD(%%YTMP3) + + ; Minimum of 256 bytes to run vectorized code + cmp %%LEN, POLY1305_BLOCK_SIZE*16 + jb %%_final_loop + + ; Spread accumulator into 44-bit limbs in quadwords + mov %%T0, %%A0 + and %%T0, [rel mask_44] ;; First limb (A[43:0]) + vmovq %%XTMP1, %%T0 + + mov %%T0, %%A1 + shrd %%A0, %%T0, 44 + and %%A0, [rel mask_44] ;; Second limb (A[77:52]) + vmovq %%XTMP2, %%A0 + + shrd %%A1, %%A2, 24 + and %%A1, [rel mask_42] ;; Third limb (A[129:88]) + vmovq %%XTMP3, %%A1 + + ; Load first block of data (64 bytes) + vmovdqu %%YTMP4, [%%MSG] + vmovdqu %%YTMP5, [%%MSG + 32] + + ; Interleave the data to form 44-bit limbs + ; + ; %%YMM_ACC0 to have bits 0-43 of all 4 blocks in 4 qwords + ; %%YMM_ACC1 to have bits 87-44 of all 4 blocks in 4 qwords + ; %%YMM_ACC2 to have bits 127-88 of all 4 blocks in 4 qwords + vpunpckhqdq %%YMM_ACC2, %%YTMP4, %%YTMP5 + vpunpcklqdq %%YMM_ACC0, %%YTMP4, %%YTMP5 + + vpsrlq %%YMM_ACC1, %%YMM_ACC0, 44 + vpsllq %%YTMP4, %%YMM_ACC2, 20 + vpor %%YMM_ACC1, %%YTMP4 + vpand %%YMM_ACC1, [rel mask_44] + + vpand %%YMM_ACC0, [rel mask_44] + vpsrlq %%YMM_ACC2, 24 + + ; Add 2^128 to all 4 final qwords of the message + vpor %%YMM_ACC2, [rel high_bit] + + vpaddq %%YMM_ACC0, %%YTMP1 + vpaddq %%YMM_ACC1, %%YTMP2 + vpaddq %%YMM_ACC2, %%YTMP3 + + ; Use memory in stack to save powers of R, before loading them into YMM registers + ; The first 16*4 bytes will contain the 16 bytes of the 4 powers of R + ; The last 32 bytes will contain the last 2 bits of powers of R, spread in 4 qwords, + ; to be OR'd with the highest qwords + vmovq %%XTMP1, %%R0 + vpinsrq %%XTMP1, %%R1, 1 + vinserti128 %%YTMP5, %%XTMP1, 1 + + vpxor %%YTMP10, %%YTMP10 + vpxor %%YTMP6, %%YTMP6 + + ; Calculate R^2 + mov %%T0, %%R1 + shr %%T0, 2 + add %%T0, %%R1 ;; T0 = R1 + (R1 >> 2) + + mov %%A0, %%R0 + mov %%A1, %%R1 + + POLY1305_MUL_REDUCE %%A0, %%A1, %%A2, %%R0, %%R1, %%T0, %%T1, %%T2, %%T3, %%GP_RAX, %%GP_RDX, no_A2 + + vmovq %%XTMP1, %%A0 + vpinsrq %%XTMP1, %%A1, 1 + vinserti128 %%YTMP5, %%XTMP1, 0 + + vmovq %%XTMP1, %%A2 + vinserti128 %%YTMP6, %%XTMP1, 0 + + ; Calculate R^3 + POLY1305_MUL_REDUCE %%A0, %%A1, %%A2, %%R0, %%R1, %%T0, %%T1, %%T2, %%T3, %%GP_RAX, %%GP_RDX + + vmovq %%XTMP1, %%A0 + vpinsrq %%XTMP1, %%A1, 1 + vinserti128 %%YTMP7, %%XTMP1, 1 + + vmovq %%XTMP1, %%A2 + vinserti128 %%YTMP2, %%XTMP1, 1 + + ; Calculate R^4 + POLY1305_MUL_REDUCE %%A0, %%A1, %%A2, %%R0, %%R1, %%T0, %%T1, %%T2, %%T3, %%GP_RAX, %%GP_RDX + + vmovq %%XTMP1, %%A0 + vpinsrq %%XTMP1, %%A1, 1 + vinserti128 %%YTMP7, %%XTMP1, 0 + + vmovq %%XTMP1, %%A2 + vinserti128 %%YTMP2, %%XTMP1, 0 + + vpunpckhqdq %%YMM_R2, %%YTMP5, %%YTMP10 + vpunpcklqdq %%YMM_R0, %%YTMP5, %%YTMP10 + vpunpckhqdq %%YTMP3, %%YTMP7, %%YTMP10 + vpunpcklqdq %%YTMP4, %%YTMP7, %%YTMP10 + + vpslldq %%YMM_R2, %%YMM_R2, 8 + vpslldq %%YTMP6, %%YTMP6, 8 + vpslldq %%YMM_R0, %%YMM_R0, 8 + vpor %%YMM_R2, %%YMM_R2, %%YTMP3 + vpor %%YMM_R0, %%YMM_R0, %%YTMP4 + vpor %%YTMP6, %%YTMP6, %%YTMP2 + + ; Move 2 MSbits to top 24 bits, to be OR'ed later + vpsllq %%YTMP6, 40 + + vpsrlq %%YMM_R1, %%YMM_R0, 44 + vpsllq %%YTMP5, %%YMM_R2, 20 + vpor %%YMM_R1, %%YTMP5 + vpand %%YMM_R1, [rel mask_44] + + vpand %%YMM_R0, [rel mask_44] + vpsrlq %%YMM_R2, 24 + + vpor %%YMM_R2, %%YTMP6 + + ; Store R^4-R for later use + vmovdqa [rsp + _r4_r1_save], %%YMM_R0 + vmovdqa [rsp + _r4_r1_save + 32], %%YMM_R1 + vmovdqa [rsp + _r4_r1_save + 32*2], %%YMM_R2 + + ; Broadcast 44-bit limbs of R^4 + mov %%T0, %%A0 + and %%T0, [rel mask_44] ;; First limb (R^4[43:0]) + vmovq XWORD(%%YMM_R0), %%T0 + vpermq %%YMM_R0, %%YMM_R0, 0x0 + + mov %%T0, %%A1 + shrd %%A0, %%T0, 44 + and %%A0, [rel mask_44] ;; Second limb (R^4[87:44]) + vmovq XWORD(%%YMM_R1), %%A0 + vpermq %%YMM_R1, %%YMM_R1, 0x0 + + shrd %%A1, %%A2, 24 + and %%A1, [rel mask_42] ;; Third limb (R^4[129:88]) + vmovq XWORD(%%YMM_R2), %%A1 + vpermq %%YMM_R2, %%YMM_R2, 0x0 + + ; Generate 4*5*R^4 + vpsllq %%YTMP1, %%YMM_R1, 2 + vpsllq %%YTMP2, %%YMM_R2, 2 + + ; 5*R^4 + vpaddq %%YTMP1, %%YMM_R1 + vpaddq %%YTMP2, %%YMM_R2 + + ; 4*5*R^4 + vpsllq %%YTMP1, 2 + vpsllq %%YTMP2, 2 + + ; Store R^4-R for later use + vmovdqa [rsp + _r4_save], %%YMM_R0 + vmovdqa [rsp + _r4_save + 32], %%YMM_R1 + vmovdqa [rsp + _r4_save + 32*2], %%YMM_R2 + vmovdqa [rsp + _r4p_save], %%YTMP1 + vmovdqa [rsp + _r4p_save + 32], %%YTMP2 + + mov %%T0, %%LEN + and %%T0, 0xffffffffffffffc0 ; multiple of 64 bytes + +%%_poly1305_blocks_loop: + cmp %%T0, POLY1305_BLOCK_SIZE*4 + jbe %%_poly1305_blocks_loop_end + + POLY1305_MSG_MUL_REDUCE_VEC4 %%YMM_ACC0, %%YMM_ACC1, %%YMM_ACC2, \ + [rsp + _r4_save], [rsp + _r4_save + 32], [rsp + _r4_save + 32*2], \ + [rsp + _r4p_save], [rsp + _r4p_save + 32], \ + %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10, %%YTMP11, %%YTMP12, \ + %%MSG, %%T0 + + jmp %%_poly1305_blocks_loop + +%%_poly1305_blocks_loop_end: + + ;; Need to multiply by r^4, r^3, r^2, r + + ; Read R^4-R + vmovdqa %%YMM_R0, [rsp + _r4_r1_save] + vmovdqa %%YMM_R1, [rsp + _r4_r1_save + 32] + vmovdqa %%YMM_R2, [rsp + _r4_r1_save + 32*2] + + ; Then multiply by r^4-r + + ; %%YTMP1 to have bits 87-44 of all 1-4th powers of R' in 4 qwords + ; %%YTMP2 to have bits 129-88 of all 1-4th powers of R' in 4 qwords + vpsllq %%YTMP10, %%YMM_R1, 2 + vpaddq %%YTMP1, %%YMM_R1, %%YTMP10 ; R1' (R1*5) + vpsllq %%YTMP10, %%YMM_R2, 2 + vpaddq %%YTMP2, %%YMM_R2, %%YTMP10 ; R2' (R2*5) + + ; 4*5*R + vpsllq %%YTMP1, 2 + vpsllq %%YTMP2, 2 + + POLY1305_MUL_REDUCE_VEC4 %%YMM_ACC0, %%YMM_ACC1, %%YMM_ACC2, \ + %%YMM_R0, %%YMM_R1, %%YMM_R2, %%YTMP1, %%YTMP2, \ + %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 + + vextracti128 XWORD(%%YTMP1), %%YMM_ACC0, 1 + vextracti128 XWORD(%%YTMP2), %%YMM_ACC1, 1 + vextracti128 XWORD(%%YTMP3), %%YMM_ACC2, 1 + + vpaddq XWORD(%%YMM_ACC0), XWORD(%%YTMP1) + vpaddq XWORD(%%YMM_ACC1), XWORD(%%YTMP2) + vpaddq XWORD(%%YMM_ACC2), XWORD(%%YTMP3) + + vpsrldq XWORD(%%YTMP1), XWORD(%%YMM_ACC0), 8 + vpsrldq XWORD(%%YTMP2), XWORD(%%YMM_ACC1), 8 + vpsrldq XWORD(%%YTMP3), XWORD(%%YMM_ACC2), 8 + + ; Finish folding and clear second qword + vpaddq XWORD(%%YMM_ACC0), XWORD(%%YTMP1) + vpaddq XWORD(%%YMM_ACC1), XWORD(%%YTMP2) + vpaddq XWORD(%%YMM_ACC2), XWORD(%%YTMP3) + vmovq XWORD(%%YMM_ACC0), XWORD(%%YMM_ACC0) + vmovq XWORD(%%YMM_ACC1), XWORD(%%YMM_ACC1) + vmovq XWORD(%%YMM_ACC2), XWORD(%%YMM_ACC2) + + add %%MSG, POLY1305_BLOCK_SIZE*4 + + and %%LEN, (POLY1305_BLOCK_SIZE*4 - 1) ; Get remaining lengths (LEN < 64 bytes) + +%%_simd_to_gp: + ; Carry propagation + vpsrlq %%XTMP1, XWORD(%%YMM_ACC0), 44 + vpand XWORD(%%YMM_ACC0), [rel mask_44] ; Clear top 20 bits + vpaddq XWORD(%%YMM_ACC1), %%XTMP1 + vpsrlq %%XTMP1, XWORD(%%YMM_ACC1), 44 + vpand XWORD(%%YMM_ACC1), [rel mask_44] ; Clear top 20 bits + vpaddq XWORD(%%YMM_ACC2), %%XTMP1 + vpsrlq %%XTMP1, XWORD(%%YMM_ACC2), 42 + vpand XWORD(%%YMM_ACC2), [rel mask_42] ; Clear top 22 bits + vpsllq %%XTMP2, %%XTMP1, 2 + vpaddq %%XTMP1, %%XTMP2 + vpaddq XWORD(%%YMM_ACC0), %%XTMP1 + + ; Put together A + vmovq %%A0, XWORD(%%YMM_ACC0) + + vmovq %%T0, XWORD(%%YMM_ACC1) + mov %%T1, %%T0 + shl %%T1, 44 + or %%A0, %%T1 + + shr %%T0, 20 + vmovq %%A2, XWORD(%%YMM_ACC2) + mov %%A1, %%A2 + shl %%A1, 24 + or %%A1, %%T0 + shr %%A2, 40 + + ; Clear powers of R +%ifdef SAFE_DATA + vpxor %%YTMP1, %%YTMP1 + vmovdqa [rsp + _r4_r1_save], %%YTMP1 + vmovdqa [rsp + _r4_r1_save + 32], %%YTMP1 + vmovdqa [rsp + _r4_r1_save + 32*2], %%YTMP1 + vmovdqa [rsp + _r4_save], %%YTMP1 + vmovdqa [rsp + _r4_save + 32], %%YTMP1 + vmovdqa [rsp + _r4_save + 32*2], %%YTMP1 + vmovdqa [rsp + _r4p_save], %%YTMP1 + vmovdqa [rsp + _r4p_save + 32], %%YTMP1 +%endif + +%%_final_loop: + cmp %%LEN, POLY1305_BLOCK_SIZE + jb %%_poly1305_blocks_partial + + ;; A += MSG[i] + add %%A0, [%%MSG + 0] + adc %%A1, [%%MSG + 8] + adc %%A2, 1 ;; no padding bit + + mov %%T0, %%R1 + shr %%T0, 2 + add %%T0, %%R1 ;; T0 = R1 + (R1 >> 2) + + POLY1305_MUL_REDUCE %%A0, %%A1, %%A2, %%R0, %%R1, \ + %%T0, %%T1, %%T2, %%T3, %%GP_RAX, %%GP_RDX + + add %%MSG, POLY1305_BLOCK_SIZE + sub %%LEN, POLY1305_BLOCK_SIZE + + jmp %%_final_loop + +%%_poly1305_blocks_partial: + + or %%LEN, %%LEN + jz %%_poly1305_blocks_exit + + simd_load_avx_16 %%XTMP1, %%MSG, %%LEN + +%ifnidn %%PAD_16,pad_to_16 + ;; pad the message + lea %%T2, [rel pad16_bit] + shl %%LEN, 4 + vpor %%XTMP1, [%%T2 + %%LEN] +%endif + vmovq %%T0, %%XTMP1 + vpextrq %%T1, %%XTMP1, 1 + ;; A += MSG[i] + add %%A0, %%T0 + adc %%A1, %%T1 +%ifnidn %%PAD_16,pad_to_16 + adc %%A2, 0 ;; no padding bit +%else + adc %%A2, 1 ;; padding bit please +%endif + + mov %%T0, %%R1 + shr %%T0, 2 + add %%T0, %%R1 ;; T0 = R1 + (R1 >> 2) + + POLY1305_MUL_REDUCE %%A0, %%A1, %%A2, %%R0, %%R1, \ + %%T0, %%T1, %%T2, %%T3, %%GP_RAX, %%GP_RDX + +%%_poly1305_blocks_exit: +%endmacro + +;; ============================================================================= +;; ============================================================================= +;; Finalizes Poly1305 hash calculation on a message +;; ============================================================================= +%macro POLY1305_FINALIZE 8 +%define %%KEY %1 ; [in] pointer to 32 byte key +%define %%MAC %2 ; [in/out] pointer to store MAC value into (16 bytes) +%define %%A0 %3 ; [in/out] accumulator bits 63..0 +%define %%A1 %4 ; [in/out] accumulator bits 127..64 +%define %%A2 %5 ; [in/out] accumulator bits 195..128 +%define %%T0 %6 ; [clobbered] GPR register +%define %%T1 %7 ; [clobbered] GPR register +%define %%T2 %8 ; [clobbered] GPR register + + ;; T = A - P, where P = 2^130 - 5 + ;; P[63..0] = 0xFFFFFFFFFFFFFFFB + ;; P[127..64] = 0xFFFFFFFFFFFFFFFF + ;; P[195..128] = 0x0000000000000003 + mov %%T0, %%A0 + mov %%T1, %%A1 + mov %%T2, %%A2 + + sub %%T0, -5 ;; 0xFFFFFFFFFFFFFFFB + sbb %%T1, -1 ;; 0xFFFFFFFFFFFFFFFF + sbb %%T2, 0x3 + + ;; if A > (2^130 - 5) then A = T + ;; - here, if borrow/CF == false then A = T + cmovnc %%A0, %%T0 + cmovnc %%A1, %%T1 + + ;; MAC = (A + S) mod 2^128 (S = key[16..31]) + add %%A0, [%%KEY + (2 * 8)] + adc %%A1, [%%KEY + (3 * 8)] + + ;; store MAC + mov [%%MAC + (0 * 8)], %%A0 + mov [%%MAC + (1 * 8)], %%A1 +%endmacro + +;; ============================================================================= +;; ============================================================================= +;; Creates stack frame and saves registers +;; ============================================================================= +%macro FUNC_ENTRY 0 + mov rax, rsp + sub rsp, STACKFRAME_size + and rsp, -32 + + mov [rsp + _gpr_save + 8*0], rbx + mov [rsp + _gpr_save + 8*1], rbp + mov [rsp + _gpr_save + 8*2], r12 + mov [rsp + _gpr_save + 8*3], r13 + mov [rsp + _gpr_save + 8*4], r14 + mov [rsp + _gpr_save + 8*5], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*6], rsi + mov [rsp + _gpr_save + 8*7], rdi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _xmm_save + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif + mov [rsp + _rsp_save], rax + +%endmacro ; FUNC_ENTRY + +;; ============================================================================= +;; ============================================================================= +;; Restores registers and removes the stack frame +;; ============================================================================= +%macro FUNC_EXIT 0 +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_all_ymms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + + mov rbx, [rsp + _gpr_save + 8*0] + mov rbp, [rsp + _gpr_save + 8*1] + mov r12, [rsp + _gpr_save + 8*2] + mov r13, [rsp + _gpr_save + 8*3] + mov r14, [rsp + _gpr_save + 8*4] + mov r15, [rsp + _gpr_save + 8*5] +%ifndef LINUX + mov rsi, [rsp + _gpr_save + 8*6] + mov rdi, [rsp + _gpr_save + 8*7] +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _xmm_save + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif + mov rsp, [rsp + _rsp_save] + +%endmacro + +;; ============================================================================= +;; ============================================================================= +;; void poly1305_aead_update_fma_avx2(const void *msg, const uint64_t msg_len, +;; void *hash, const void *key) +;; arg1 - Input message +;; arg2 - Message length +;; arg3 - Input/output hash +;; arg4 - Poly1305 key +align 32 +MKGLOBAL(poly1305_aead_update_fma_avx2,function,internal) +poly1305_aead_update_fma_avx2: + +%ifdef SAFE_PARAM + or arg1, arg1 + jz .poly1305_update_exit + + or arg3, arg3 + jz .poly1305_update_exit + + or arg4, arg4 + jz .poly1305_update_exit +%endif + + FUNC_ENTRY + +%ifdef LINUX +%xdefine _a0 gp3 +%xdefine _a1 gp4 +%xdefine _a2 gp5 +%xdefine _r0 gp6 +%xdefine _r1 gp7 +%xdefine _len arg2 +%xdefine _arg3 arg4 ; use rcx, arg3 = rdx +%else +%xdefine _a0 gp3 +%xdefine _a1 rdi +%xdefine _a2 gp5 ; = arg4 / r9 +%xdefine _r0 gp6 +%xdefine _r1 gp7 +%xdefine _len gp2 ; rsi +%xdefine _arg3 arg3 ; arg +%endif + + ;; load R + mov _r0, [arg4 + 0 * 8] + mov _r1, [arg4 + 1 * 8] + + ;; load accumulator / current hash value + ;; note: arg4 can't be used beyond this point +%ifdef LINUX + mov _arg3, arg3 ; note: _arg3 = arg4 (linux) +%endif + mov _a0, [_arg3 + 0 * 8] + mov _a1, [_arg3 + 1 * 8] + mov _a2, [_arg3 + 2 * 8] ; note: _a2 = arg4 (win) + +%ifndef LINUX + mov _len, arg2 ;; arg2 = rdx on Windows +%endif + POLY1305_BLOCKS arg1, _len, _a0, _a1, _a2, _r0, _r1, \ + gp10, gp11, gp8, gp9, rax, rdx, pad_to_16 + + ;; save accumulator back + mov [_arg3 + 0 * 8], _a0 + mov [_arg3 + 1 * 8], _a1 + mov [_arg3 + 2 * 8], _a2 + + FUNC_EXIT +.poly1305_update_exit: + ret + +;; ============================================================================= +;; ============================================================================= +;; void poly1305_aead_complete_fma_avx2(const void *hash, const void *key, +;; void *tag) +;; arg1 - Input hash +;; arg2 - Poly1305 key +;; arg3 - Output tag +align 32 +MKGLOBAL(poly1305_aead_complete_fma_avx2,function,internal) +poly1305_aead_complete_fma_avx2: + +%ifdef SAFE_PARAM + or arg1, arg1 + jz .poly1305_complete_exit + + or arg2, arg2 + jz .poly1305_complete_exit + + or arg3, arg3 + jz .poly1305_complete_exit +%endif + + FUNC_ENTRY + +%xdefine _a0 gp6 +%xdefine _a1 gp7 +%xdefine _a2 gp8 + + ;; load accumulator / current hash value + mov _a0, [arg1 + 0 * 8] + mov _a1, [arg1 + 1 * 8] + mov _a2, [arg1 + 2 * 8] + + POLY1305_FINALIZE arg2, arg3, _a0, _a1, _a2, gp9, gp10, gp11 + + ;; clear Poly key +%ifdef SAFE_DATA + vpxor xmm0, xmm0 + vmovdqu [arg2], ymm0 +%endif + + FUNC_EXIT +.poly1305_complete_exit: + ret + +;; ============================================================================= +;; ============================================================================= +;; void poly1305_mac_fma_avx2(IMB_JOB *job) +;; arg1 - job structure +align 32 +MKGLOBAL(poly1305_mac_fma_avx2,function,internal) +poly1305_mac_fma_avx2: + FUNC_ENTRY + +%ifndef LINUX + mov job, arg1 +%endif + +%ifdef SAFE_PARAM + or job, job + jz .poly1305_mac_exit +%endif + +%xdefine _a0 gp1 +%xdefine _a1 gp2 +%xdefine _a2 gp3 +%xdefine _r0 gp4 +%xdefine _r1 gp5 + + mov gp6, [job + _poly1305_key] + POLY1305_INIT gp6, _a0, _a1, _a2, _r0, _r1 + + mov msg, [job + _src] + add msg, [job + _hash_start_src_offset_in_bytes] + mov len, [job + _msg_len_to_hash] + POLY1305_BLOCKS msg, len, _a0, _a1, _a2, _r0, _r1, \ + gp6, gp7, gp8, gp9, rax, rdx, no_padding + + mov rax, [job + _poly1305_key] + mov rdx, [job + _auth_tag_output] + POLY1305_FINALIZE rax, rdx, _a0, _a1, _a2, gp6, gp7, gp8 + +.poly1305_mac_exit: + FUNC_EXIT + ret + +mksection stack-noexec diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c deleted file mode 100644 index a6bc6c635fd3a0c79fbcd1fbdc075ce0bf6942d4..0000000000000000000000000000000000000000 --- a/lib/avx512/mb_mgr_avx512.c +++ /dev/null @@ -1,2106 +0,0 @@ -/******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -#include -#include -#include - -#define AVX512 -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_zmms - -#include "ipsec-mb.h" -#include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" -#include "include/zuc_internal.h" -#include "include/snow3g.h" -#include "include/gcm.h" -#include "include/chacha20_poly1305.h" -#include "include/snow3g_submit.h" - -#include "include/save_xmms.h" -#include "include/asm.h" -#include "include/des.h" -#include "include/gcm.h" -#include "include/cpu_feature.h" -#include "include/noaesni.h" -#include "include/aesni_emu.h" -#include "include/error.h" - -IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *aes_cntr_ccm_128_vaes_avx512(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_256_vaes_avx512(IMB_JOB *job); - -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -void aes_cmac_256_subkey_gen_avx512(const void *key_exp, - void *key1, void *key2); -uint32_t hec_32_avx(const uint8_t *in); -uint64_t hec_64_avx(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *snow_v_avx(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); - - -IMB_JOB *submit_job_snow3g_uea2_vaes_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uea2_vaes_avx512(MB_MGR_SNOW3G_OOO *state); - -IMB_JOB *submit_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state); - -IMB_JOB *submit_job_snow3g_uia2_vaes_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uia2_vaes_avx512(MB_MGR_SNOW3G_OOO *state); - -IMB_JOB *submit_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state); - -#define SAVE_XMMS save_xmms_avx -#define RESTORE_XMMS restore_xmms_avx - -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx512 -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx512 -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx512 - -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx512 -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx512 -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx512 - -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx512 -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx512 -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx512 - -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx - -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx512 -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx512 - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx512 -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx512 -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx512 -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx512 -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx512 -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx512 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx512 -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx512 - -#define AES_CBC_DEC_128 aes_cbc_dec_128_avx512 -#define AES_CBC_DEC_192 aes_cbc_dec_192_avx512 -#define AES_CBC_DEC_256 aes_cbc_dec_256_avx512 - -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx - -#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx512 -#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx512 - -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx - -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx512 -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx512 -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx512 -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx512 - -#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx512 -#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx512 - -#define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 -#define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 - -#define SUBMIT_JOB_DES_CBC_DEC submit_job_des_cbc_dec_avx512 -#define FLUSH_JOB_DES_CBC_DEC flush_job_des_cbc_dec_avx512 - -#define SUBMIT_JOB_3DES_CBC_ENC submit_job_3des_cbc_enc_avx512 -#define FLUSH_JOB_3DES_CBC_ENC flush_job_3des_cbc_enc_avx512 - -#define SUBMIT_JOB_3DES_CBC_DEC submit_job_3des_cbc_dec_avx512 -#define FLUSH_JOB_3DES_CBC_DEC flush_job_3des_cbc_dec_avx512 - -#define SUBMIT_JOB_DOCSIS_DES_ENC submit_job_docsis_des_enc_avx512 -#define FLUSH_JOB_DOCSIS_DES_ENC flush_job_docsis_des_enc_avx512 - -#define SUBMIT_JOB_DOCSIS_DES_DEC submit_job_docsis_des_dec_avx512 -#define FLUSH_JOB_DOCSIS_DES_DEC flush_job_docsis_des_dec_avx512 - -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512 - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx512 -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx512 -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx512 -#define POLY1305_MAC poly1305_mac_avx512 - -#define SUBMIT_JOB_SNOW_V snow_v_avx -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx - -IMB_JOB *submit_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - - -IMB_JOB *submit_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); - -static IMB_JOB *submit_snow3g_uea2_job_vaes_avx512(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - if ((job->msg_len_to_cipher_in_bits & 7) || - (job->cipher_start_offset_in_bits & 7)) - return def_submit_snow3g_uea2_job(state, job); - - return submit_job_snow3g_uea2_vaes_avx512(snow3g_uea2_ooo, job); -} - -static IMB_JOB *flush_snow3g_uea2_job_vaes_avx512(IMB_MGR *state) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - return flush_job_snow3g_uea2_vaes_avx512(snow3g_uea2_ooo); -} - -static IMB_JOB *submit_snow3g_uea2_job_avx512(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - if ((job->msg_len_to_cipher_in_bits & 7) || - (job->cipher_start_offset_in_bits & 7)) - return def_submit_snow3g_uea2_job(state, job); - - return submit_job_snow3g_uea2_avx512(snow3g_uea2_ooo, job); -} - -static IMB_JOB *flush_snow3g_uea2_job_avx512(IMB_MGR *state) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - return flush_job_snow3g_uea2_avx512(snow3g_uea2_ooo); -} - -static IMB_JOB *(*submit_job_snow3g_uea2_avx512_ptr) - (IMB_MGR *state, IMB_JOB *job) = - submit_snow3g_uea2_job_avx512; - -static IMB_JOB *(*flush_job_snow3g_uea2_avx512_ptr)(IMB_MGR *state) = - flush_snow3g_uea2_job_avx512; - -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_avx512_ptr -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_avx512_ptr - -static IMB_JOB *(*submit_job_snow3g_uia2_avx512_ptr) - (MB_MGR_SNOW3G_OOO *state, IMB_JOB *job) = - submit_job_snow3g_uia2_avx512; - -static IMB_JOB *(*flush_job_snow3g_uia2_avx512_ptr) - (MB_MGR_SNOW3G_OOO *state) = flush_job_snow3g_uia2_avx512; - -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_avx512_ptr -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_avx512_ptr - - -/* ====================================================================== */ - -IMB_JOB *submit_job_chacha20_enc_dec_avx512(IMB_JOB *job); - -void poly1305_mac_fma_avx512(IMB_JOB *job); -void poly1305_mac_plain_avx512(IMB_JOB *job); - -static void (*poly1305_mac_avx512) - (IMB_JOB *) = poly1305_mac_plain_avx512; - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, - const uint64_t key_size); -__forceinline -IMB_JOB * -FLUSH_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, - const uint64_t key_size); - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, - const uint64_t key_size); - -#define SUBMIT_JOB_HMAC submit_job_hmac_avx512 -#define FLUSH_JOB_HMAC flush_job_hmac_avx512 -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx512 -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx512 -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx512 -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx512 -#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx512 -#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx512 -#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx512 -#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx512 -#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 -#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 - -#define AES_GCM_DEC_128 aes_gcm_dec_128_avx512 -#define AES_GCM_ENC_128 aes_gcm_enc_128_avx512 -#define AES_GCM_DEC_192 aes_gcm_dec_192_avx512 -#define AES_GCM_ENC_192 aes_gcm_enc_192_avx512 -#define AES_GCM_DEC_256 aes_gcm_dec_256_avx512 -#define AES_GCM_ENC_256 aes_gcm_enc_256_avx512 - -#define AES_GCM_DEC_128_VAES aes_gcm_dec_128_vaes_avx512 -#define AES_GCM_ENC_128_VAES aes_gcm_enc_128_vaes_avx512 -#define AES_GCM_DEC_192_VAES aes_gcm_dec_192_vaes_avx512 -#define AES_GCM_ENC_192_VAES aes_gcm_enc_192_vaes_avx512 -#define AES_GCM_DEC_256_VAES aes_gcm_dec_256_vaes_avx512 -#define AES_GCM_ENC_256_VAES aes_gcm_enc_256_vaes_avx512 - -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx512 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx512 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx512 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx512 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx512 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx512 - -#define AES_GCM_DEC_IV_128_VAES aes_gcm_dec_var_iv_128_vaes_avx512 -#define AES_GCM_ENC_IV_128_VAES aes_gcm_enc_var_iv_128_vaes_avx512 -#define AES_GCM_DEC_IV_192_VAES aes_gcm_dec_var_iv_192_vaes_avx512 -#define AES_GCM_ENC_IV_192_VAES aes_gcm_enc_var_iv_192_vaes_avx512 -#define AES_GCM_DEC_IV_256_VAES aes_gcm_dec_var_iv_256_vaes_avx512 -#define AES_GCM_ENC_IV_256_VAES aes_gcm_enc_var_iv_256_vaes_avx512 - -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx512 -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx512 - -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_avx512 -#define FLUSH_JOB flush_job_avx512 -#define QUEUE_SIZE queue_size_avx512 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512 -#define GET_NEXT_JOB get_next_job_avx512 -#define GET_COMPLETED_JOB get_completed_job_avx512 - -/* ====================================================================== */ - -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512 -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512 - -/* ====================================================================== */ - -#define AES_CFB_128_ONE aes_cfb_128_one_avx512 -#define AES_CFB_256_ONE aes_cfb_256_one_avx512 - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx512 -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx512 - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx512 -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx512 - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx512 -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx512 - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx512 -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx512 - -/* ====================================================================== */ - -extern uint32_t -ethernet_fcs_avx512_local(const void *msg, const uint64_t len, - const void *tag_ouput); -extern uint32_t -ethernet_fcs_avx_local(const void *msg, const uint64_t len, - const void *tag_ouput); - -#define ETHERNET_FCS ethernet_fcs_avx_local - -extern uint32_t ethernet_fcs_avx512(const void *msg, const uint64_t len); -extern uint32_t crc16_x25_avx512(const void *msg, const uint64_t len); -extern uint32_t crc32_sctp_avx512(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_a_avx512(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_b_avx512(const void *msg, const uint64_t len); -extern uint32_t crc16_fp_data_avx512(const void *msg, const uint64_t len); -extern uint32_t crc11_fp_header_avx512(const void *msg, const uint64_t len); -extern uint32_t crc7_fp_header_avx512(const void *msg, const uint64_t len); -extern uint32_t crc10_iuup_data_avx512(const void *msg, const uint64_t len); -extern uint32_t crc6_iuup_header_avx512(const void *msg, const uint64_t len); -extern uint32_t -crc32_wimax_ofdma_data_avx512(const void *msg, const uint64_t len); -extern uint32_t -crc8_wimax_ofdma_hcs_avx512(const void *msg, const uint64_t len); - -extern uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); -extern uint32_t crc16_x25_avx(const void *msg, const uint64_t len); -extern uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len); -extern uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len); -extern uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len); -extern uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len); -extern uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len); -extern uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); -extern uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); -extern uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); - -extern void snow3g_f9_1_buffer_vaes_avx512(const snow3g_key_schedule_t *pHandle, - const void *pIV, - const void *pBufferIn, - const uint64_t lengthInBits, - void *pDigest); - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx512 -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx512 -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx512 -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx512 - -/* ====================================================================== */ - -/* - * GCM submit / flush API for AVX512 arch - */ -static IMB_JOB * -plain_submit_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -plain_submit_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -vaes_submit_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) - AES_GCM_DEC_IV_128_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_GCM_DEC_IV_192_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else /* assume 32 bytes */ - AES_GCM_DEC_IV_256_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -vaes_submit_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) - AES_GCM_ENC_IV_128_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_GCM_ENC_IV_192_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else /* assume 32 bytes */ - AES_GCM_ENC_IV_256_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB *(*submit_job_aes_gcm_enc_avx512) - (IMB_MGR *state, IMB_JOB *job) = plain_submit_gcm_enc_avx512; - -static IMB_JOB *(*submit_job_aes_gcm_dec_avx512) - (IMB_MGR *state, IMB_JOB *job) = plain_submit_gcm_dec_avx512; - -static IMB_JOB *(*submit_job_aes_cntr_avx512) - (IMB_JOB *job) = submit_job_aes_cntr_avx; -static IMB_JOB *(*submit_job_aes_cntr_bit_avx512) - (IMB_JOB *job) = submit_job_aes_cntr_bit_avx; - -static IMB_JOB *(*submit_job_pon_enc_avx512) - (IMB_JOB *job) = submit_job_pon_enc_avx; -static IMB_JOB *(*submit_job_pon_dec_avx512) - (IMB_JOB *job) = submit_job_pon_dec_avx; -static IMB_JOB *(*submit_job_pon_enc_no_ctr_avx512) - (IMB_JOB *job) = submit_job_pon_enc_no_ctr_avx; -static IMB_JOB *(*submit_job_pon_dec_no_ctr_avx512) - (IMB_JOB *job) = submit_job_pon_dec_no_ctr_avx; - -static IMB_JOB * -vaes_submit_cntr_avx512(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_128_submit_vaes_avx512(job); - else if (24 == job->key_len_in_bytes) - aes_cntr_192_submit_vaes_avx512(job); - else /* assume 32 bytes */ - aes_cntr_256_submit_vaes_avx512(job); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -static IMB_JOB * -vaes_submit_cntr_bit_avx512(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_submit_vaes_avx512(job); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_submit_vaes_avx512(job); - else /* assume 32 bytes */ - aes_cntr_bit_256_submit_vaes_avx512(job); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -/* ====================================================================== */ - -static IMB_JOB * -(*submit_job_aes128_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = submit_job_aes128_enc_avx; - -static IMB_JOB * -(*submit_job_aes192_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = submit_job_aes192_enc_avx; - -static IMB_JOB * -(*submit_job_aes256_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = submit_job_aes256_enc_avx; - -static IMB_JOB * -(*flush_job_aes128_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes128_enc_avx; - -static IMB_JOB * -(*flush_job_aes192_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes192_enc_avx; - -static IMB_JOB * -(*flush_job_aes256_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes256_enc_avx; - -static void -(*aes_cbc_dec_128_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = aes_cbc_dec_128_avx; -static void -(*aes_cbc_dec_192_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = aes_cbc_dec_192_avx; -static void -(*aes_cbc_dec_256_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = aes_cbc_dec_256_avx; - -static IMB_JOB * -(*submit_job_aes128_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state, - IMB_JOB *job) = submit_job_aes128_cmac_auth_avx; - -static IMB_JOB * -(*flush_job_aes128_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state) = flush_job_aes128_cmac_auth_avx; - -static IMB_JOB * -(*submit_job_aes256_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state, - IMB_JOB *job) = submit_job_aes256_cmac_auth_avx; - -static IMB_JOB * -(*flush_job_aes256_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state) = flush_job_aes256_cmac_auth_avx; - -static IMB_JOB * -(*submit_job_aes128_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state, - IMB_JOB *job) = submit_job_aes128_ccm_auth_avx; - -static IMB_JOB * -(*flush_job_aes128_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state) = flush_job_aes128_ccm_auth_avx; - -static IMB_JOB * -(*submit_job_aes256_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state, - IMB_JOB *job) = submit_job_aes256_ccm_auth_avx; - -static IMB_JOB * -(*flush_job_aes256_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state) = flush_job_aes256_ccm_auth_avx; - -static IMB_JOB * -(*aes_cntr_ccm_128_avx512) (IMB_JOB *job) = aes_cntr_ccm_128_avx; - -static IMB_JOB * -(*aes_cntr_ccm_256_avx512) (IMB_JOB *job) = aes_cntr_ccm_256_avx; - -static IMB_JOB * -(*submit_job_zuc_eea3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eea3_no_gfni_avx512; - -static IMB_JOB * -(*flush_job_zuc_eea3_avx512) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eea3_no_gfni_avx512; - -static IMB_JOB * -(*submit_job_zuc256_eea3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc256_eea3_no_gfni_avx512; - -static IMB_JOB * -(*flush_job_zuc256_eea3_avx512) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc256_eea3_no_gfni_avx512; - -static IMB_JOB * -(*submit_job_zuc_eia3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eia3_no_gfni_avx512; - -static IMB_JOB * -(*flush_job_zuc_eia3_avx512) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eia3_no_gfni_avx512; - -static IMB_JOB * -(*submit_job_zuc256_eia3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz) = - submit_job_zuc256_eia3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc256_eia3_avx512) - (MB_MGR_ZUC_OOO *state, const uint64_t tag_sz) = - flush_job_zuc256_eia3_no_gfni_sse; - -static IMB_JOB * -(*submit_job_aes_xcbc_avx512) - (MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job) = submit_job_aes_xcbc_avx; - -static IMB_JOB * -(*flush_job_aes_xcbc_avx512) - (MB_MGR_AES_XCBC_OOO *state) = flush_job_aes_xcbc_avx; - - -static IMB_JOB * -(*submit_job_aes128_cbcs_1_9_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = - submit_job_aes128_cbcs_1_9_enc_avx; - -static IMB_JOB * -(*flush_job_aes128_cbcs_1_9_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes128_cbcs_1_9_enc_avx; - -static void -(*aes_cbcs_1_9_dec_128_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, - void *next_iv) = aes_cbcs_1_9_dec_128_avx; - -/* ====================================================================== */ - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS128_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); - -extern void aes_docsis128_dec_crc32_avx512(IMB_JOB *job); -extern void aes_docsis128_dec_crc32_vaes_avx512(IMB_JOB *job); - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS256_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); - -extern void aes_docsis256_dec_crc32_avx512(IMB_JOB *job); -extern void aes_docsis256_dec_crc32_vaes_avx512(IMB_JOB *job); - -static IMB_JOB * -submit_aes_docsis128_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) -{ - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS128_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis128_dec_crc32_avx512(job); - - return job; -} - -static IMB_JOB * -submit_aes_docsis256_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) -{ - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS256_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis256_dec_crc32_avx512(job); - - return job; -} - -extern IMB_JOB * -submit_job_aes_docsis128_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis128_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state); - -extern IMB_JOB * -submit_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state); - -extern IMB_JOB * -submit_job_aes_docsis128_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis128_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state); - -extern IMB_JOB * -submit_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state); - -static IMB_JOB * -submit_job_docsis128_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) -{ - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS128_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis128_dec_crc32_vaes_avx512(job); - - return job; -} - -static IMB_JOB * -submit_job_docsis256_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) -{ - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS256_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis256_dec_crc32_vaes_avx512(job); - - return job; -} - -static IMB_JOB * -(*submit_job_docsis128_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_job_aes_docsis128_enc_crc32_avx512; - -static IMB_JOB * -(*submit_job_docsis256_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_job_aes_docsis256_enc_crc32_avx512; - -static IMB_JOB * -(*flush_job_docsis128_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state) = - flush_job_aes_docsis128_enc_crc32_avx512; - -static IMB_JOB * -(*flush_job_docsis256_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state) = - flush_job_aes_docsis256_enc_crc32_avx512; - -static IMB_JOB * -(*submit_job_docsis128_sec_crc_dec_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_aes_docsis128_dec_crc32_avx512; - -static IMB_JOB * -(*submit_job_docsis256_sec_crc_dec_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_aes_docsis256_dec_crc32_avx512; - -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC submit_job_docsis128_sec_crc_enc_fn -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC submit_job_docsis256_sec_crc_enc_fn -#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC flush_job_docsis128_sec_crc_enc_fn -#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC flush_job_docsis256_sec_crc_enc_fn -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC submit_job_docsis128_sec_crc_dec_fn -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC submit_job_docsis256_sec_crc_dec_fn - - -/* ====================================================================== */ - -static void -reset_ooo_mgrs(IMB_MGR *state) -{ - unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; - MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; - MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; - MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; - MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; - MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; - MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; - - /* Init AES out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - memset(aes128_ooo->lens, 0, - sizeof(aes128_ooo->lens)); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xFEDCBA9876543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0, - sizeof(aes192_ooo->lens)); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xFEDCBA9876543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(aes256_ooo->lens, 0, - sizeof(aes256_ooo->lens)); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xFEDCBA9876543210; - aes256_ooo->num_lanes_inuse = 0; - } else { - /* init 8 lanes */ - memset(aes128_ooo->lens, 0xFF, - sizeof(aes128_ooo->lens)); - memset(&aes128_ooo->lens[0], 0, - sizeof(aes128_ooo->lens[0]) * 8); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xF76543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0xFF, - sizeof(aes192_ooo->lens)); - memset(&aes192_ooo->lens[0], 0, - sizeof(aes192_ooo->lens[0]) * 8); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xF76543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(aes256_ooo->lens, 0xFF, - sizeof(aes256_ooo->lens)); - memset(&aes256_ooo->lens[0], 0, - sizeof(aes256_ooo->lens[0]) * 8); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xF76543210; - aes256_ooo->num_lanes_inuse = 0; - } - - - /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) - * uses same settings as AES CBC. - */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - memset(docsis128_sec_ooo->lens, 0, - sizeof(docsis128_sec_ooo->lens)); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0, - sizeof(docsis256_sec_ooo->lens)); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - } else { - /* init 8 lanes */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(&docsis128_sec_ooo->lens[0], 0, - sizeof(docsis128_sec_ooo->lens[0]) * 8); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xF76543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(&docsis256_sec_ooo->lens[0], 0, - sizeof(docsis256_sec_ooo->lens[0]) * 8); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xF76543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - } - - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - memset(docsis128_crc32_sec_ooo->lens, 0, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; - } else { - /* init 8 lanes */ - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(&docsis128_crc32_sec_ooo->lens[0], 0, - sizeof(docsis128_crc32_sec_ooo->lens[0]) * 8); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(&docsis256_crc32_sec_ooo->lens[0], 0, - sizeof(docsis256_crc32_sec_ooo->lens[0]) * 8); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; - } - - /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ - /* - separate DES OOO for encryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des_enc_ooo->lens[j] = 0; - des_enc_ooo->job_in_lane[j] = NULL; - } - des_enc_ooo->unused_lanes = 0xFEDCBA9876543210; - des_enc_ooo->num_lanes_inuse = 0; - memset(&des_enc_ooo->args, 0, sizeof(des_enc_ooo->args)); - - /* - separate DES OOO for decryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des_dec_ooo->lens[j] = 0; - des_dec_ooo->job_in_lane[j] = NULL; - } - des_dec_ooo->unused_lanes = 0xFEDCBA9876543210; - des_dec_ooo->num_lanes_inuse = 0; - memset(&des_dec_ooo->args, 0, sizeof(des_dec_ooo->args)); - - /* - separate 3DES OOO for encryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des3_enc_ooo->lens[j] = 0; - des3_enc_ooo->job_in_lane[j] = NULL; - } - des3_enc_ooo->unused_lanes = 0xFEDCBA9876543210; - des3_enc_ooo->num_lanes_inuse = 0; - memset(&des3_enc_ooo->args, 0, sizeof(des3_enc_ooo->args)); - - /* - separate 3DES OOO for decryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des3_dec_ooo->lens[j] = 0; - des3_dec_ooo->job_in_lane[j] = NULL; - } - des3_dec_ooo->unused_lanes = 0xFEDCBA9876543210; - des3_dec_ooo->num_lanes_inuse = 0; - memset(&des3_dec_ooo->args, 0, sizeof(des3_dec_ooo->args)); - - /* - separate DOCSIS DES OOO for encryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - docsis_des_enc_ooo->lens[j] = 0; - docsis_des_enc_ooo->job_in_lane[j] = NULL; - } - docsis_des_enc_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis_des_enc_ooo->num_lanes_inuse = 0; - memset(&docsis_des_enc_ooo->args, 0, - sizeof(docsis_des_enc_ooo->args)); - - /* - separate DES OOO for decryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - docsis_des_dec_ooo->lens[j] = 0; - docsis_des_dec_ooo->job_in_lane[j] = NULL; - } - docsis_des_dec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis_des_dec_ooo->num_lanes_inuse = 0; - memset(&docsis_des_dec_ooo->args, 0, - sizeof(docsis_des_dec_ooo->args)); - - /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFEDCBA9876543210; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0xffff; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFEDCBA9876543210; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0xffff; - memset(zuc_eia3_ooo->args.digest, 0, sizeof(zuc_eia3_ooo->args.digest)); - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFEDCBA9876543210; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0xffff; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; - memset(zuc256_eia3_ooo->args.digest, 0, - sizeof(zuc256_eia3_ooo->args.digest)); - - /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0; - hmac_sha_1_ooo->lens[5] = 0; - hmac_sha_1_ooo->lens[6] = 0; - hmac_sha_1_ooo->lens[7] = 0; - hmac_sha_1_ooo->lens[8] = 0; - hmac_sha_1_ooo->lens[9] = 0; - hmac_sha_1_ooo->lens[10] = 0; - hmac_sha_1_ooo->lens[11] = 0; - hmac_sha_1_ooo->lens[12] = 0; - hmac_sha_1_ooo->lens[13] = 0; - hmac_sha_1_ooo->lens[14] = 0; - hmac_sha_1_ooo->lens[15] = 0; - hmac_sha_1_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_sha_1_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX512_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5 * 4] = 0x80; - p[64 - 2] = 0x02; - p[64 - 1] = 0xA0; - } - - /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0; - hmac_sha_224_ooo->lens[5] = 0; - hmac_sha_224_ooo->lens[6] = 0; - hmac_sha_224_ooo->lens[7] = 0; - hmac_sha_224_ooo->lens[8] = 0; - hmac_sha_224_ooo->lens[9] = 0; - hmac_sha_224_ooo->lens[10] = 0; - hmac_sha_224_ooo->lens[11] = 0; - hmac_sha_224_ooo->lens[12] = 0; - hmac_sha_224_ooo->lens[13] = 0; - hmac_sha_224_ooo->lens[14] = 0; - hmac_sha_224_ooo->lens[15] = 0; - hmac_sha_224_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_sha_224_ooo->num_lanes_inuse = 0; - /* sha256 and sha224 are very similar except for - * digest constants and output size - */ - for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7 * 4] = 0x80; /* digest 7 words long */ - p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ - p[64 - 1] = 0xE0; - } - - /* Init HMAC/SHA256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0; - hmac_sha_256_ooo->lens[5] = 0; - hmac_sha_256_ooo->lens[6] = 0; - hmac_sha_256_ooo->lens[7] = 0; - hmac_sha_256_ooo->lens[8] = 0; - hmac_sha_256_ooo->lens[9] = 0; - hmac_sha_256_ooo->lens[10] = 0; - hmac_sha_256_ooo->lens[11] = 0; - hmac_sha_256_ooo->lens[12] = 0; - hmac_sha_256_ooo->lens[13] = 0; - hmac_sha_256_ooo->lens[14] = 0; - hmac_sha_256_ooo->lens[15] = 0; - hmac_sha_256_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_sha_256_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - /* hmac related */ - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); - p[8 * 4] = 0x80; /* 8 digest words */ - p[64 - 2] = 0x03; /* length */ - p[64 - 1] = 0x00; - } - - /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0; - hmac_sha_384_ooo->lens[3] = 0; - hmac_sha_384_ooo->lens[4] = 0; - hmac_sha_384_ooo->lens[5] = 0; - hmac_sha_384_ooo->lens[6] = 0; - hmac_sha_384_ooo->lens[7] = 0; - hmac_sha_384_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 384 bits, 1408 bits == 0x0580. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } - - /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0; - hmac_sha_512_ooo->lens[3] = 0; - hmac_sha_512_ooo->lens[4] = 0; - hmac_sha_512_ooo->lens[5] = 0; - hmac_sha_512_ooo->lens[6] = 0; - hmac_sha_512_ooo->lens[7] = 0; - hmac_sha_512_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 512 bits, 1536 bits == 0x600. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } - - /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0; - hmac_md5_ooo->lens[9] = 0; - hmac_md5_ooo->lens[10] = 0; - hmac_md5_ooo->lens[11] = 0; - hmac_md5_ooo->lens[12] = 0; - hmac_md5_ooo->lens[13] = 0; - hmac_md5_ooo->lens[14] = 0; - hmac_md5_ooo->lens[15] = 0; - hmac_md5_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_md5_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX512_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4 * 4] = 0x80; - p[64 - 7] = 0x02; - p[64 - 8] = 0x80; - } - - /* Init AES/XCBC OOO fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - aes_xcbc_ooo->unused_lanes = 0xFEDCBA9876543210; - aes_xcbc_ooo->num_lanes_inuse = 0; - for (j = 0; j < 16; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, - 0x00, 15); - } - } else { - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xF76543210; - aes_xcbc_ooo->num_lanes_inuse = 0; - for (j = 0; j < 8; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, - 0x00, 15); - } - } - - /* Init AES-CCM auth out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - memset(aes_ccm_ooo->init_done, 0, - sizeof(aes_ccm_ooo->init_done)); - memset(aes_ccm_ooo->lens, 0, - sizeof(aes_ccm_ooo->lens)); - memset(aes_ccm_ooo->job_in_lane, 0, - sizeof(aes_ccm_ooo->job_in_lane)); - aes_ccm_ooo->unused_lanes = 0xFEDCBA9876543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - memset(aes256_ccm_ooo->init_done, 0, - sizeof(aes256_ccm_ooo->init_done)); - memset(aes256_ccm_ooo->lens, 0, - sizeof(aes256_ccm_ooo->lens)); - memset(aes256_ccm_ooo->job_in_lane, 0, - sizeof(aes256_ccm_ooo->job_in_lane)); - aes256_ccm_ooo->unused_lanes = 0xFEDCBA9876543210; - aes256_ccm_ooo->num_lanes_inuse = 0; - - } else { - /* init 8 lanes */ - for (j = 0; j < 8; j++) { - aes_ccm_ooo->init_done[j] = 0; - aes_ccm_ooo->lens[j] = 0; - aes_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes_ccm_ooo->lens[j] = 0xFFFF; - - aes_ccm_ooo->unused_lanes = 0xF76543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_ccm_ooo->init_done[j] = 0; - aes256_ccm_ooo->lens[j] = 0; - aes256_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes256_ccm_ooo->lens[j] = 0xFFFF; - - aes256_ccm_ooo->unused_lanes = 0xF76543210; - aes256_ccm_ooo->num_lanes_inuse = 0; - } - - /* Init AES-CMAC auth out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - memset(aes_cmac_ooo->init_done, 0, - sizeof(aes_cmac_ooo->init_done)); - memset(aes_cmac_ooo->lens, 0, - sizeof(aes_cmac_ooo->lens)); - memset(aes_cmac_ooo->job_in_lane, 0, - sizeof(aes_cmac_ooo->job_in_lane)); - aes_cmac_ooo->unused_lanes = 0xFEDCBA9876543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - memset(aes256_cmac_ooo->init_done, 0, - sizeof(aes256_cmac_ooo->init_done)); - memset(aes256_cmac_ooo->lens, 0, - sizeof(aes256_cmac_ooo->lens)); - memset(aes256_cmac_ooo->job_in_lane, 0, - sizeof(aes256_cmac_ooo->job_in_lane)); - aes256_cmac_ooo->unused_lanes = 0xFEDCBA9876543210; - aes256_cmac_ooo->num_lanes_inuse = 0; - } else { - /* init 8 lanes */ - memset(aes_cmac_ooo->init_done, 0, - sizeof(aes_cmac_ooo->init_done)); - memset(aes_cmac_ooo->lens, 0xFF, - sizeof(aes_cmac_ooo->lens)); - memset(&aes_cmac_ooo->lens[0], 0, - sizeof(aes_cmac_ooo->lens[0]) * 8); - memset(aes_cmac_ooo->job_in_lane, 0, - sizeof(aes_cmac_ooo->job_in_lane)); - aes_cmac_ooo->unused_lanes = 0xF76543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - memset(aes256_cmac_ooo->init_done, 0, - sizeof(aes256_cmac_ooo->init_done)); - memset(aes256_cmac_ooo->lens, 0xFF, - sizeof(aes256_cmac_ooo->lens)); - memset(&aes256_cmac_ooo->lens[0], 0, - sizeof(aes256_cmac_ooo->lens[0]) * 8); - memset(aes256_cmac_ooo->job_in_lane, 0, - sizeof(aes256_cmac_ooo->job_in_lane)); - aes256_cmac_ooo->unused_lanes = 0xF76543210; - aes256_cmac_ooo->num_lanes_inuse = 0; - } - - /* Init AES CBC-S out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 12 lanes */ - memset(aes128_cbcs_ooo->lens64, 0xFF, - sizeof(aes128_cbcs_ooo->lens64)); - memset(&aes128_cbcs_ooo->lens64[0], 0, - sizeof(aes128_cbcs_ooo->lens64[0]) * 12); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xBA9876543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; - - } else { - memset(aes128_cbcs_ooo->lens, 0xFF, - sizeof(aes128_cbcs_ooo->lens)); - memset(&aes128_cbcs_ooo->lens[0], 0, - sizeof(aes128_cbcs_ooo->lens[0]) * 8); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xF76543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; - } - - /* Init SNOW3G out-of-order fields */ - memset(&snow3g_uea2_ooo->args, 0, - sizeof(snow3g_uea2_ooo->args)); - memset(snow3g_uea2_ooo->job_in_lane, 0, - sizeof(snow3g_uea2_ooo->job_in_lane)); - snow3g_uea2_ooo->unused_lanes = 0xFEDCBA9876543210; - snow3g_uea2_ooo->num_lanes_inuse = 0; - snow3g_uea2_ooo->init_mask = 0; - snow3g_uea2_ooo->init_done = 0; - memset(snow3g_uea2_ooo->lens, 0xff, - sizeof(snow3g_uea2_ooo->lens)); - memset(&snow3g_uia2_ooo->args, 0, - sizeof(snow3g_uia2_ooo->args)); - memset(snow3g_uia2_ooo->job_in_lane, 0, - sizeof(snow3g_uia2_ooo->job_in_lane)); - memset(snow3g_uia2_ooo->ks, 0, - sizeof(snow3g_uia2_ooo->ks)); - snow3g_uia2_ooo->unused_lanes = 0xFEDCBA9876543210; - snow3g_uia2_ooo->num_lanes_inuse = 0; - snow3g_uia2_ooo->init_mask = 0; - snow3g_uia2_ooo->init_done = 0; - memset(snow3g_uia2_ooo->lens, 0, - sizeof(snow3g_uia2_ooo->lens)); -} - -IMB_DLL_LOCAL void -init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) -{ -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_AVX512; - - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - aes_cbc_dec_128_avx512 = aes_cbc_dec_128_vaes_avx512; - aes_cbc_dec_192_avx512 = aes_cbc_dec_192_vaes_avx512; - aes_cbc_dec_256_avx512 = aes_cbc_dec_256_vaes_avx512; - submit_job_aes128_enc_avx512 = - submit_job_aes128_enc_vaes_avx512; - flush_job_aes128_enc_avx512 = - flush_job_aes128_enc_vaes_avx512; - submit_job_aes192_enc_avx512 = - submit_job_aes192_enc_vaes_avx512; - flush_job_aes192_enc_avx512 = - flush_job_aes192_enc_vaes_avx512; - submit_job_aes256_enc_avx512 = - submit_job_aes256_enc_vaes_avx512; - flush_job_aes256_enc_avx512 = - flush_job_aes256_enc_vaes_avx512; - submit_job_aes128_cmac_auth_avx512 = - submit_job_aes128_cmac_auth_vaes_avx512; - flush_job_aes128_cmac_auth_avx512 = - flush_job_aes128_cmac_auth_vaes_avx512; - submit_job_aes256_cmac_auth_avx512 = - submit_job_aes256_cmac_auth_vaes_avx512; - flush_job_aes256_cmac_auth_avx512 = - flush_job_aes256_cmac_auth_vaes_avx512; - submit_job_aes128_ccm_auth_avx512 = - submit_job_aes128_ccm_auth_vaes_avx512; - flush_job_aes128_ccm_auth_avx512 = - flush_job_aes128_ccm_auth_vaes_avx512; - submit_job_aes256_ccm_auth_avx512 = - submit_job_aes256_ccm_auth_vaes_avx512; - flush_job_aes256_ccm_auth_avx512 = - flush_job_aes256_ccm_auth_vaes_avx512; - aes_cntr_ccm_128_avx512 = aes_cntr_ccm_128_vaes_avx512; - aes_cntr_ccm_256_avx512 = aes_cntr_ccm_256_vaes_avx512; - - submit_job_docsis128_sec_crc_enc_fn = - submit_job_aes_docsis128_enc_crc32_vaes_avx512; - submit_job_docsis256_sec_crc_enc_fn = - submit_job_aes_docsis256_enc_crc32_vaes_avx512; - flush_job_docsis128_sec_crc_enc_fn = - flush_job_aes_docsis128_enc_crc32_vaes_avx512; - flush_job_docsis256_sec_crc_enc_fn = - flush_job_aes_docsis256_enc_crc32_vaes_avx512; - - submit_job_docsis128_sec_crc_dec_fn = - submit_job_docsis128_sec_crc_dec_vaes_avx512; - submit_job_docsis256_sec_crc_dec_fn = - submit_job_docsis256_sec_crc_dec_vaes_avx512; - - submit_job_aes_xcbc_avx512 = submit_job_aes_xcbc_vaes_avx512; - flush_job_aes_xcbc_avx512 = flush_job_aes_xcbc_vaes_avx512; - - submit_job_aes128_cbcs_1_9_enc_avx512 = - submit_job_aes128_cbcs_1_9_enc_vaes_avx512; - flush_job_aes128_cbcs_1_9_enc_avx512 = - flush_job_aes128_cbcs_1_9_enc_vaes_avx512; - aes_cbcs_1_9_dec_128_avx512 = aes_cbcs_1_9_dec_128_vaes_avx512; - - submit_job_snow3g_uea2_avx512_ptr = - submit_snow3g_uea2_job_vaes_avx512; - flush_job_snow3g_uea2_avx512_ptr = - flush_snow3g_uea2_job_vaes_avx512; - } - - if ((state->features & IMB_FEATURE_GFNI) && - (state->features & IMB_FEATURE_VAES)) { - submit_job_zuc_eea3_avx512 = submit_job_zuc_eea3_gfni_avx512; - flush_job_zuc_eea3_avx512 = flush_job_zuc_eea3_gfni_avx512; - submit_job_zuc_eia3_avx512 = submit_job_zuc_eia3_gfni_avx512; - flush_job_zuc_eia3_avx512 = flush_job_zuc_eia3_gfni_avx512; - submit_job_zuc256_eea3_avx512 = - submit_job_zuc256_eea3_gfni_avx512; - flush_job_zuc256_eea3_avx512 = - flush_job_zuc256_eea3_gfni_avx512; - submit_job_zuc256_eia3_avx512 = - submit_job_zuc256_eia3_gfni_sse; - flush_job_zuc256_eia3_avx512 = - flush_job_zuc256_eia3_gfni_sse; - } - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set handlers */ - state->get_next_job = get_next_job_avx512; - state->submit_job = submit_job_avx512; - state->submit_job_nocheck = submit_job_nocheck_avx512; - state->get_completed_job = get_completed_job_avx512; - state->flush_job = flush_job_avx512; - state->queue_size = queue_size_avx512; - state->keyexp_128 = aes_keyexp_128_avx512; - state->keyexp_192 = aes_keyexp_192_avx512; - state->keyexp_256 = aes_keyexp_256_avx512; - state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx512; - state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx512; - state->xcbc_keyexp = aes_xcbc_expand_key_avx512; - state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx512; - state->sha1 = sha1_avx512; - state->sha224_one_block = sha224_one_block_avx512; - state->sha224 = sha224_avx512; - state->sha256_one_block = sha256_one_block_avx512; - state->sha256 = sha256_avx512; - state->sha384_one_block = sha384_one_block_avx512; - state->sha384 = sha384_avx512; - state->sha512_one_block = sha512_one_block_avx512; - state->sha512 = sha512_avx512; - state->md5_one_block = md5_one_block_avx512; - state->aes128_cfb_one = aes_cfb_128_one_avx512; - - state->eea3_1_buffer = zuc_eea3_1_buffer_avx512; - state->eea3_4_buffer = zuc_eea3_4_buffer_avx; - state->eia3_1_buffer = zuc_eia3_1_buffer_avx512; - - if ((state->features & IMB_FEATURE_GFNI) && - (state->features & IMB_FEATURE_VAES)) { - state->eea3_n_buffer = zuc_eea3_n_buffer_gfni_avx512; - state->eia3_n_buffer = zuc_eia3_n_buffer_gfni_avx512; - } else { - state->eea3_n_buffer = zuc_eea3_n_buffer_avx512; - state->eia3_n_buffer = zuc_eia3_n_buffer_avx512; - } - - state->f8_1_buffer = kasumi_f8_1_buffer_avx; - state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; - state->f8_2_buffer = kasumi_f8_2_buffer_avx; - state->f8_3_buffer = kasumi_f8_3_buffer_avx; - state->f8_4_buffer = kasumi_f8_4_buffer_avx; - state->f8_n_buffer = kasumi_f8_n_buffer_avx; - state->f9_1_buffer = kasumi_f9_1_buffer_avx; - state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; - state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; - state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; - state->kasumi_key_sched_size = kasumi_key_sched_size_avx; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx512; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx512; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx512; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx512; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx512; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx512; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx512; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx512; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx512; - state->snow3g_init_key_sched = snow3g_init_key_sched_avx512; - state->snow3g_key_sched_size = snow3g_key_sched_size_avx512; - - state->hec_32 = hec_32_avx; - state->hec_64 = hec_64_avx; - state->crc32_ethernet_fcs = ethernet_fcs_avx; - state->crc16_x25 = crc16_x25_avx; - state->crc32_sctp = crc32_sctp_avx; - state->crc24_lte_a = crc24_lte_a_avx; - state->crc24_lte_b = crc24_lte_b_avx; - state->crc16_fp_data = crc16_fp_data_avx; - state->crc11_fp_header = crc11_fp_header_avx; - state->crc7_fp_header = crc7_fp_header_avx; - state->crc10_iuup_data = crc10_iuup_data_avx; - state->crc6_iuup_header = crc6_iuup_header_avx; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; - - if ((state->features & IMB_FEATURE_VPCLMULQDQ) == - IMB_FEATURE_VPCLMULQDQ) { - state->crc32_ethernet_fcs = ethernet_fcs_avx512; - state->crc16_x25 = crc16_x25_avx512; - state->crc32_sctp = crc32_sctp_avx512; - state->crc24_lte_a = crc24_lte_a_avx512; - state->crc24_lte_b = crc24_lte_b_avx512; - state->crc16_fp_data = crc16_fp_data_avx512; - state->crc11_fp_header = crc11_fp_header_avx512; - state->crc7_fp_header = crc7_fp_header_avx512; - state->crc10_iuup_data = crc10_iuup_data_avx512; - state->crc6_iuup_header = crc6_iuup_header_avx512; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx512; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx512; -#ifndef _WIN32 - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_vaes_avx512; -#endif - } - - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - submit_job_aes_cntr_avx512 = vaes_submit_cntr_avx512; - submit_job_aes_cntr_bit_avx512 = vaes_submit_cntr_bit_avx512; - submit_job_pon_enc_avx512 = submit_job_pon_enc_vaes_avx512; - submit_job_pon_enc_no_ctr_avx512 = - submit_job_pon_enc_no_ctr_vaes_avx512; - submit_job_pon_dec_avx512 = submit_job_pon_dec_vaes_avx512; - submit_job_pon_dec_no_ctr_avx512 = - submit_job_pon_dec_no_ctr_vaes_avx512; - submit_job_snow3g_uea2_avx512_ptr = - submit_snow3g_uea2_job_vaes_avx512; - flush_job_snow3g_uea2_avx512_ptr = - flush_snow3g_uea2_job_vaes_avx512; - } - - if (state->features & IMB_FEATURE_AVX512_IFMA) { - poly1305_mac_avx512 = poly1305_mac_fma_avx512; - state->chacha20_poly1305_init = - init_chacha20_poly1305_fma_avx512; - state->chacha20_poly1305_enc_update = - update_enc_chacha20_poly1305_fma_avx512; - state->chacha20_poly1305_dec_update = - update_dec_chacha20_poly1305_fma_avx512; - state->chacha20_poly1305_finalize = - finalize_chacha20_poly1305_fma_avx512; - } else { - state->chacha20_poly1305_init = init_chacha20_poly1305_avx512; - state->chacha20_poly1305_enc_update = - update_enc_chacha20_poly1305_avx512; - state->chacha20_poly1305_dec_update = - update_dec_chacha20_poly1305_avx512; - state->chacha20_poly1305_finalize = - finalize_chacha20_poly1305_avx512; - } - - if ((state->features & (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) == - (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) { - state->gcm128_enc = aes_gcm_enc_128_vaes_avx512; - state->gcm192_enc = aes_gcm_enc_192_vaes_avx512; - state->gcm256_enc = aes_gcm_enc_256_vaes_avx512; - state->gcm128_dec = aes_gcm_dec_128_vaes_avx512; - state->gcm192_dec = aes_gcm_dec_192_vaes_avx512; - state->gcm256_dec = aes_gcm_dec_256_vaes_avx512; - state->gcm128_init = aes_gcm_init_128_vaes_avx512; - state->gcm192_init = aes_gcm_init_192_vaes_avx512; - state->gcm256_init = aes_gcm_init_256_vaes_avx512; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_vaes_avx512; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_vaes_avx512; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_vaes_avx512; - state->gcm128_enc_update = aes_gcm_enc_128_update_vaes_avx512; - state->gcm192_enc_update = aes_gcm_enc_192_update_vaes_avx512; - state->gcm256_enc_update = aes_gcm_enc_256_update_vaes_avx512; - state->gcm128_dec_update = aes_gcm_dec_128_update_vaes_avx512; - state->gcm192_dec_update = aes_gcm_dec_192_update_vaes_avx512; - state->gcm256_dec_update = aes_gcm_dec_256_update_vaes_avx512; - state->gcm128_enc_finalize = - aes_gcm_enc_128_finalize_vaes_avx512; - state->gcm192_enc_finalize = - aes_gcm_enc_192_finalize_vaes_avx512; - state->gcm256_enc_finalize = - aes_gcm_enc_256_finalize_vaes_avx512; - state->gcm128_dec_finalize = - aes_gcm_dec_128_finalize_vaes_avx512; - state->gcm192_dec_finalize = - aes_gcm_dec_192_finalize_vaes_avx512; - state->gcm256_dec_finalize = - aes_gcm_dec_256_finalize_vaes_avx512; - state->gcm128_precomp = aes_gcm_precomp_128_vaes_avx512; - state->gcm192_precomp = aes_gcm_precomp_192_vaes_avx512; - state->gcm256_precomp = aes_gcm_precomp_256_vaes_avx512; - state->gcm128_pre = aes_gcm_pre_128_vaes_avx512; - state->gcm192_pre = aes_gcm_pre_192_vaes_avx512; - state->gcm256_pre = aes_gcm_pre_256_vaes_avx512; - state->ghash = ghash_vaes_avx512; - state->ghash_pre = ghash_pre_vaes_avx512; - - submit_job_aes_gcm_enc_avx512 = vaes_submit_gcm_enc_avx512; - submit_job_aes_gcm_dec_avx512 = vaes_submit_gcm_dec_avx512; - - state->gmac128_init = imb_aes_gmac_init_128_vaes_avx512; - state->gmac192_init = imb_aes_gmac_init_192_vaes_avx512; - state->gmac256_init = imb_aes_gmac_init_256_vaes_avx512; - state->gmac128_update = imb_aes_gmac_update_128_vaes_avx512; - state->gmac192_update = imb_aes_gmac_update_192_vaes_avx512; - state->gmac256_update = imb_aes_gmac_update_256_vaes_avx512; - state->gmac128_finalize = imb_aes_gmac_finalize_128_vaes_avx512; - state->gmac192_finalize = imb_aes_gmac_finalize_192_vaes_avx512; - state->gmac256_finalize = imb_aes_gmac_finalize_256_vaes_avx512; - - submit_job_snow3g_uia2_avx512_ptr = - submit_job_snow3g_uia2_vaes_avx512; - flush_job_snow3g_uia2_avx512_ptr = - flush_job_snow3g_uia2_vaes_avx512; - } else { - state->gcm128_enc = aes_gcm_enc_128_avx512; - state->gcm192_enc = aes_gcm_enc_192_avx512; - state->gcm256_enc = aes_gcm_enc_256_avx512; - state->gcm128_dec = aes_gcm_dec_128_avx512; - state->gcm192_dec = aes_gcm_dec_192_avx512; - state->gcm256_dec = aes_gcm_dec_256_avx512; - state->gcm128_init = aes_gcm_init_128_avx512; - state->gcm192_init = aes_gcm_init_192_avx512; - state->gcm256_init = aes_gcm_init_256_avx512; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx512; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx512; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx512; - state->gcm128_enc_update = aes_gcm_enc_128_update_avx512; - state->gcm192_enc_update = aes_gcm_enc_192_update_avx512; - state->gcm256_enc_update = aes_gcm_enc_256_update_avx512; - state->gcm128_dec_update = aes_gcm_dec_128_update_avx512; - state->gcm192_dec_update = aes_gcm_dec_192_update_avx512; - state->gcm256_dec_update = aes_gcm_dec_256_update_avx512; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx512; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx512; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx512; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx512; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx512; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx512; - state->gcm128_precomp = aes_gcm_precomp_128_avx512; - state->gcm192_precomp = aes_gcm_precomp_192_avx512; - state->gcm256_precomp = aes_gcm_precomp_256_avx512; - state->gcm128_pre = aes_gcm_pre_128_avx512; - state->gcm192_pre = aes_gcm_pre_192_avx512; - state->gcm256_pre = aes_gcm_pre_256_avx512; - state->ghash = ghash_avx512; - state->ghash_pre = ghash_pre_avx_gen2; - - state->gmac128_init = imb_aes_gmac_init_128_avx512; - state->gmac192_init = imb_aes_gmac_init_192_avx512; - state->gmac256_init = imb_aes_gmac_init_256_avx512; - state->gmac128_update = imb_aes_gmac_update_128_avx512; - state->gmac192_update = imb_aes_gmac_update_192_avx512; - state->gmac256_update = imb_aes_gmac_update_256_avx512; - state->gmac128_finalize = imb_aes_gmac_finalize_128_avx512; - state->gmac192_finalize = imb_aes_gmac_finalize_192_avx512; - state->gmac256_finalize = imb_aes_gmac_finalize_256_avx512; - } - -} - -void -init_mb_mgr_avx512(IMB_MGR *state) -{ - init_mb_mgr_avx512_internal(state, 1); -} - -#include "mb_mgr_code.h" diff --git a/lib/avx512_t1/aes128_gcm_by8_avx512.asm b/lib/avx512_t1/aes128_gcm_by8_avx512.asm index 4415ca93f29fd126023e6903395b9eb889e4fc25..5487a4feed403aa0000e2ef33942fb31ab0a0347 100644 --- a/lib/avx512_t1/aes128_gcm_by8_avx512.asm +++ b/lib/avx512_t1/aes128_gcm_by8_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512_t1/gcm_avx512.asm" +%include "include/gcm_avx512.inc" diff --git a/lib/avx512_t1/aes192_gcm_by8_avx512.asm b/lib/avx512_t1/aes192_gcm_by8_avx512.asm index 96ae8046af7c42a89e2eade05ed02c137e35741b..9a1e645ffa52fed3f2c3cbc65ec03779ce88e168 100644 --- a/lib/avx512_t1/aes192_gcm_by8_avx512.asm +++ b/lib/avx512_t1/aes192_gcm_by8_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512_t1/gcm_avx512.asm" +%include "include/gcm_avx512.inc" diff --git a/lib/avx512_t1/aes256_gcm_by8_avx512.asm b/lib/avx512_t1/aes256_gcm_by8_avx512.asm index 5083081ac3160863f8f8490867ebc62d0b1c45b0..ea7728b707a906fe81da4f3918ed64976f61865d 100644 --- a/lib/avx512_t1/aes256_gcm_by8_avx512.asm +++ b/lib/avx512_t1/aes256_gcm_by8_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512_t1/gcm_avx512.asm" +%include "include/gcm_avx512.inc" diff --git a/lib/avx512_t1/chacha20_avx512.asm b/lib/avx512_t1/chacha20_avx512.asm index 845e531d3f38379d08de82f1c8f89b3aa8b9934e..e38042a503c5717ce5eb7042a30d957dff271db9 100644 --- a/lib/avx512_t1/chacha20_avx512.asm +++ b/lib/avx512_t1/chacha20_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,14 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" %include "include/const.inc" -%include "include/reg_sizes.asm" -%include "include/transpose_avx512.asm" -%include "include/aes_common.asm" -%include "include/chacha_poly_defines.asm" +%include "include/reg_sizes.inc" +%include "include/transpose_avx512.inc" +%include "include/aes_common.inc" +%include "include/chacha_poly_defines.inc" %include "include/cet.inc" mksection .rodata default rel diff --git a/lib/avx512_t1/des_x16_avx512.asm b/lib/avx512_t1/des_x16_avx512.asm index 3dedfa2a35b3dab3d5ee1cc46879bbbb883b2aa7..c940dd8d1ba5a4f6b497882c2eb00efb233a4be5 100644 --- a/lib/avx512_t1/des_x16_avx512.asm +++ b/lib/avx512_t1/des_x16_avx512.asm @@ -1,5 +1,5 @@ ; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -46,13 +46,13 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31 and K1 to K7 -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/constants.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/constants.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/clear_regs.inc" %ifdef LINUX %define arg1 rdi diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index ba04cb5b7bfa903cc9e821ca4c88c874faa3c529..9c3c64b75a0996b7c82bfa3c009f8691e681652f 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t1/mb_mgr_avx512_t1.c b/lib/avx512_t1/mb_mgr_avx512_t1.c index 3dbbc17820e201c08de9ad2c9ea9ac27d85b3bab..c5db36d2d5526423550875bdf1efb6351c5196fb 100644 --- a/lib/avx512_t1/mb_mgr_avx512_t1.c +++ b/lib/avx512_t1/mb_mgr_avx512_t1.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -52,6 +52,7 @@ #include "include/arch_avx2_type1.h" /* MD5 */ #include "include/arch_avx512_type1.h" #include "include/arch_avx512_type2.h" +#include "include/arch_sse_type1.h" #include "include/ooo_mgr_reset.h" @@ -73,7 +74,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512_t1 #define SUBMIT_HASH_BURST submit_hash_burst_avx512_t1 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx512_t1 - +#define SET_SUITE_ID_FN set_suite_id_avx512_t1 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512 @@ -334,12 +335,18 @@ submit_aes_docsis256_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, return job; } -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC submit_job_aes_docsis128_enc_crc32_avx512 -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC submit_job_aes_docsis256_enc_crc32_avx512 -#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC flush_job_aes_docsis128_enc_crc32_avx512 -#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC flush_job_aes_docsis256_enc_crc32_avx512 -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC submit_aes_docsis128_dec_crc32_avx512 -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC submit_aes_docsis256_dec_crc32_avx512 +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC \ + submit_job_aes_docsis128_enc_crc32_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC \ + submit_job_aes_docsis256_enc_crc32_avx512 +#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC \ + flush_job_aes_docsis128_enc_crc32_avx512 +#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC \ + flush_job_aes_docsis256_enc_crc32_avx512 +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC \ + submit_aes_docsis128_dec_crc32_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC \ + submit_aes_docsis256_dec_crc32_avx512 /* ====================================================================== */ @@ -350,7 +357,7 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_ooo, 8); ooo_mgr_aes_reset(state->aes192_ooo, 8); ooo_mgr_aes_reset(state->aes256_ooo, 8); - + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) * uses same settings as AES CBC. */ @@ -464,6 +471,7 @@ init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_avx512; state->keyexp_192 = aes_keyexp_192_avx512; @@ -487,6 +495,7 @@ init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_avx512; state->aes128_cfb_one = aes_cfb_128_one_avx512; + state->aes256_cfb_one = aes_cfb_256_one_avx512; state->eea3_1_buffer = zuc_eea3_1_buffer_avx512; state->eea3_4_buffer = zuc_eea3_4_buffer_avx; @@ -574,7 +583,7 @@ init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs) state->gcm256_pre = aes_gcm_pre_256_avx512; state->ghash = ghash_avx512; - state->ghash_pre = ghash_pre_avx_gen2; + state->ghash_pre = ghash_pre_avx512; state->gmac128_init = imb_aes_gmac_init_128_avx512; state->gmac192_init = imb_aes_gmac_init_192_avx512; @@ -585,6 +594,9 @@ init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_avx512; state->gmac192_finalize = imb_aes_gmac_finalize_192_avx512; state->gmac256_finalize = imb_aes_gmac_finalize_256_avx512; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_avx; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_avx; } #include "mb_mgr_code.h" diff --git a/lib/avx512_t1/mb_mgr_des_avx512.asm b/lib/avx512_t1/mb_mgr_des_avx512.asm index 9a36428fac5e08db6a1270aa51240087c4861d07..efdb0067d647d10ebfaa7547372a6068c3c27756 100644 --- a/lib/avx512_t1/mb_mgr_des_avx512.asm +++ b/lib/avx512_t1/mb_mgr_des_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -41,13 +41,13 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31, K1-7 (K1-2 and K4-6 here but DES underneath clobbers K1-7). -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/constants.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/constants.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" %include "include/const.inc" extern docsis_des_x16_enc_avx512 diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm index 3cc4be23296625a842ba410fe86cb58b5c6ff20b..4f8b97c9b2e81e7e18934c243837ad59e5b77b72 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -40,14 +40,14 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31 -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ;; %define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha1_x16_avx512 diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm index 947994bb03c2686302668121d2a7c04548c576aa..c9d723ff529ad03d8d26d82232a2820374ae81f9 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -40,15 +40,15 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31 -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" ;; %define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" %use smartalign diff --git a/lib/avx512_t1/mb_mgr_hmac_sha224_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha224_flush_avx512.asm index b7467c311de005c75ce036529db2347cdba7f29d..b740bb44e9d80706ffada1c040ce4ea8eba565b5 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha224_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha224_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t1/mb_mgr_hmac_sha224_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha224_submit_avx512.asm index d8828b4553a0326114a7fb678c143b9920a67c9c..a3db8f73861b9aef09a6549dc633bae0f5e15539 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha224_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha224_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm index e449e94f496c29c4a2c66eebeccf3a48bb1f5b55..01c5ce034f53d0f628c1437c8bac9c9b4a059ca9 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -40,13 +40,13 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31 -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ;; %define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha256_x16_avx512 diff --git a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm index 0660b19d204e0aa3788d84e2d7b69d0a7a638dcf..454ea930f124f9106a19ec90dde9dec99d5d3990 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -40,15 +40,15 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31 -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" ;; %define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" %use smartalign diff --git a/lib/avx512_t1/mb_mgr_hmac_sha384_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha384_flush_avx512.asm index 70037f6db6b9b673dd65e08e67f8dfacee1366f2..3184d8e1ff6e689448c7ce53a6860f4f328b6266 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha384_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha384_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t1/mb_mgr_hmac_sha384_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha384_submit_avx512.asm index 2fc8063cc7d45f21b09a4cb44c2d79e79b17f43b..998dd4ef5240543f293bef1bd9b8f3fd8225a718 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha384_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha384_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm index 9b825d68601ffbb4b468e078889df567d54d7069..2904a0163fc3b17b15e28a8b4c713a0861110909 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -32,11 +32,11 @@ ;; ;; Clobbers ZMM0-31 -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" extern sha512_x8_avx512 diff --git a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm index 504815cc831856c5e487da928cb1c0b8e0b921ef..277def8c9efaff4f4e14490d4b12be1326c5f518 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -32,13 +32,13 @@ ;; ;; Clobbers ZMM0-31 -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %use smartalign diff --git a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm index be795516c31cd19dde5e94cc680a16e553dd9d84..0e21b2fb32c706732a0eec323894d249dfdab538 100644 --- a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,13 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/constants.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/constants.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifndef SUBMIT_JOB_ZUC128_EEA3 %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_no_gfni_avx512 @@ -331,7 +331,7 @@ mksection .text %%return_submit_eea3: %ifdef SAFE_DATA - clear_all_zmms_asm + clear_scratch_zmms_asm %else vzeroupper %endif @@ -554,7 +554,7 @@ mksection .text %endif %ifdef SAFE_DATA - clear_all_zmms_asm + clear_scratch_zmms_asm %else vzeroupper %endif @@ -701,6 +701,8 @@ FLUSH_JOB_ZUC256_EEA3: call ZUC_KEYGEN_16 + RESERVE_STACK_SPACE 5 + ; Digest 64 bytes of data lea arg1, [%%OOO + _zuc_args_digest] lea arg2, [%%OOO + _zuc_args_KS] @@ -710,6 +712,8 @@ FLUSH_JOB_ZUC256_EEA3: call ZUC_ROUND64B + RESTORE_STACK_SPACE 5 + sub %%REMAIN_BITS, 64*8 jmp %%_exit @@ -989,7 +993,7 @@ FLUSH_JOB_ZUC256_EEA3: %%return_submit_eia3: %ifdef SAFE_DATA - clear_all_zmms_asm + clear_scratch_zmms_asm %else vzeroupper %endif @@ -1190,13 +1194,12 @@ FLUSH_JOB_ZUC256_EEA3: %endif %ifdef SAFE_DATA - clear_all_zmms_asm + clear_scratch_zmms_asm %else vzeroupper %endif %%return_flush_eia3: - mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/avx512_t1/poly_avx512.asm b/lib/avx512_t1/poly_avx512.asm index 400dade99f22c195b235a97caa6fc3ef0c86abab..c21b93b536d3849d1e187fd620e223f0140e29e2 100644 --- a/lib/avx512_t1/poly_avx512.asm +++ b/lib/avx512_t1/poly_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" -%include "include/imb_job.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" [bits 64] default rel diff --git a/lib/avx512_t1/sha1_x16_avx512.asm b/lib/avx512_t1/sha1_x16_avx512.asm index afa52c5827b908c83111d507477e2ab86ba58a6e..80f648c3659650c3927dacd31ce460fb80ec0cac 100644 --- a/lib/avx512_t1/sha1_x16_avx512.asm +++ b/lib/avx512_t1/sha1_x16_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -37,13 +37,13 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/transpose_avx512.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/transpose_avx512.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" mksection .rodata default rel align 64 diff --git a/lib/avx512_t1/sha256_x16_avx512.asm b/lib/avx512_t1/sha256_x16_avx512.asm index fb060b9dfbfde7534623327d0a0e899f5b32721c..209100b7fd00fdafe81266e837ca8037b29da496 100644 --- a/lib/avx512_t1/sha256_x16_avx512.asm +++ b/lib/avx512_t1/sha256_x16_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -37,13 +37,13 @@ ;; ----------------------------------------------------------- ;; Clobbers ZMM0-31 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/transpose_avx512.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/transpose_avx512.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ; re-use K256 from sha256_oct_avx2.asm extern K256 diff --git a/lib/avx512_t1/sha512_x8_avx512.asm b/lib/avx512_t1/sha512_x8_avx512.asm index 24c03f8a9c02d5f3ee00944fa6718dfa1f53ab73..11e8f8e1fd911406a1928be63b308add2b1eb8a5 100644 --- a/lib/avx512_t1/sha512_x8_avx512.asm +++ b/lib/avx512_t1/sha512_x8_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -39,12 +39,12 @@ ;; code to compute quad SHA512 using AVX512 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/transpose_avx512.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/transpose_avx512.inc" +%include "include/clear_regs.inc" %define APPEND(a,b) a %+ b %ifdef LINUX diff --git a/lib/avx512_t1/sha_avx512.c b/lib/avx512_t1/sha_avx512.c new file mode 100644 index 0000000000000000000000000000000000000000..39014b21eeb1a2297511d6eda8ee8a0132a7c6d3 --- /dev/null +++ b/lib/avx512_t1/sha_avx512.c @@ -0,0 +1,114 @@ +/******************************************************************************* + Copyright (c) 2020-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_generic.h" +#include "include/arch_avx512_type1.h" + +/* ========================================================================== */ +/* One block SHA1 computation for IPAD / OPAD usage only */ +void sha1_one_block_avx512(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 1 /* SHA1 */); +} + +/* ========================================================================== */ +/* + * SHA1 API for use in HMAC-SHA1 when key is longer than the block size + */ +void sha1_avx512(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 1, IMB_SHA1_BLOCK_SIZE, + SHA1_PAD_SIZE); +} + +/* ========================================================================== */ +/* One block SHA224 computation for IPAD / OPAD usage only */ +void sha224_one_block_avx512(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 224 /* SHA224 */); +} + +/* ========================================================================== */ +/* + * SHA224 API for use in HMAC-SHA224 when key is longer than the block size + */ +void sha224_avx512(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE); +} + +/* ========================================================================== */ +/* One block SHA256 computation for IPAD / OPAD usage only */ +void sha256_one_block_avx512(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 256 /* SHA256 */); +} + +/* ========================================================================== */ +/* + * SHA256 API for use in HMAC-SHA256 when key is longer than the block size + */ +void sha256_avx512(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE); +} + +/* ========================================================================== */ +/* One block SHA384 computation for IPAD / OPAD usage only */ +void sha384_one_block_avx512(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 384 /* SHA384 */); +} + +/* ========================================================================== */ +/* + * SHA384 API for use in HMAC-SHA384 when key is longer than the block size + */ +void sha384_avx512(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 384, + IMB_SHA_384_BLOCK_SIZE, SHA384_PAD_SIZE); +} + +/* ========================================================================== */ +/* One block SHA512 computation for IPAD / OPAD usage only */ +void sha512_one_block_avx512(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_AVX, 512 /* SHA512 */); +} + +/* ========================================================================== */ +/* + * SHA512 API for use in HMAC-SHA512 when key is longer than the block size + */ +void sha512_avx512(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_AVX, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE); +} diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index 6b283fc2adbca491ed0fc6caf15e83a1be0b8af7..85ae8b40e0ba120e17b173c2e8971bbb32e98705 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t1/snow3g_avx512.c b/lib/avx512_t1/snow3g_avx512.c index 36d9eed73b5e0ca8472044d6e0870f89b9602cc9..188b2c1ff0adf86aab9292668590792b39408b6f 100644 --- a/lib/avx512_t1/snow3g_avx512.c +++ b/lib/avx512_t1/snow3g_avx512.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2021-2022, Intel Corporation + Copyright (c) 2021-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t1/zuc_top_avx512.c b/lib/avx512_t1/zuc_top_avx512.c index 918d25318d5a69badfdbb2e42ac81c7f7b72d3d2..79ed2b8c5bce332ba38ac8a04dc0a149228e2e63 100644 --- a/lib/avx512_t1/zuc_top_avx512.c +++ b/lib/avx512_t1/zuc_top_avx512.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -606,7 +606,7 @@ void _zuc_eia3_1_buffer_avx512(const void *pKey, asm_Eia3Round64BAVX512(&T, &keyStream[0], pIn8); /* Copy the last keystream generated * to the first 64 bytes */ - memcpy(&keyStream[0], &keyStream[16], 64); + memmove(&keyStream[0], &keyStream[16], 64); pIn8 = &pIn8[ZUC_KEYSTR_LEN]; } @@ -715,7 +715,7 @@ void _zuc_eia3_16_buffer_avx512(const void * const pKey[NUM_AVX512_BUFS], * to be in contiguous memory */ for (j = 0; j < 8; j++) - memcpy(keyStr32 + j*4, + memmove(keyStr32 + j*4, &keyStr[get_start_key_addr(i) + j*16], 16); /* If remaining bits are more than 56 bytes, we need to generate @@ -756,7 +756,7 @@ void _zuc_eia3_16_buffer_avx512(const void * const pKey[NUM_AVX512_BUFS], asm_Eia3Round64BAVX512(&T[i], &keyStr32[0], pIn8[i]); /* Copy the last keystream generated * to the first 64 bytes */ - memcpy(keyStr32, &keyStr32[16], 64); + memmove(keyStr32, &keyStr32[16], 64); pIn8[i] = &pIn8[i][ZUC_KEYSTR_LEN]; } diff --git a/lib/avx512_t1/zuc_x16_avx512.asm b/lib/avx512_t1/zuc_x16_avx512.asm index 0b1a26d90d5091cc94c364a125152690b3aaa5d6..974cd553ddb26abbb2b4621bd58a1f5915ff6db9 100644 --- a/lib/avx512_t1/zuc_x16_avx512.asm +++ b/lib/avx512_t1/zuc_x16_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/zuc_sbox.inc" -%include "include/transpose_avx512.asm" +%include "include/transpose_avx512.inc" %include "include/const.inc" -%include "include/mb_mgr_datastruct.asm" +%include "include/mb_mgr_datastruct.inc" %include "include/cet.inc" %define APPEND(a,b) a %+ b %define APPEND3(a,b,c) a %+ b %+ c diff --git a/lib/avx512_t2/aes128_gcm_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes128_gcm_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes128_gcm_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes128_gcm_api_vaes_avx512.asm index 90dbca2a355592b14b619f274983b29384c1d771..5134d9cc0fa8f6f7ddff3bcc158ead28b0d231d5 100644 --- a/lib/avx512_t2/aes128_gcm_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes128_gcm_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512_t2/gcm_api_vaes_avx512.inc" +%include "include/gcm_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes128_gcm_by48_sgl_api_vaes_avx512.asm b/lib/avx512_t2/aes128_gcm_sgl_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes128_gcm_by48_sgl_api_vaes_avx512.asm rename to lib/avx512_t2/aes128_gcm_sgl_api_vaes_avx512.asm index cd55ad0cea470a129c5910a11f87b68d0ae723c0..34961612fb97c650536af3466b1bedc8e1bff8b9 100644 --- a/lib/avx512_t2/aes128_gcm_by48_sgl_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes128_gcm_sgl_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512_t2/gcm_sgl_api_vaes_avx512.inc" +%include "include/gcm_sgl_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes128_gmac_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes128_gmac_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes128_gmac_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes128_gmac_api_vaes_avx512.asm index 7915728b86975c7dc4b3238c79ac11100edafd69..099484ca5aca6c95bd862dfbfbc6497d7ed8801f 100644 --- a/lib/avx512_t2/aes128_gmac_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes128_gmac_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512_t2/gcm_gmac_api_vaes_avx512.inc" +%include "include/gcm_gmac_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes192_gcm_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes192_gcm_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes192_gcm_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes192_gcm_api_vaes_avx512.asm index 05693ed933ba933fb41b5db4f0af00141451838d..47ec33da804b3b202c7219bc47da0f9a008f0572 100644 --- a/lib/avx512_t2/aes192_gcm_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes192_gcm_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512_t2/gcm_api_vaes_avx512.inc" +%include "include/gcm_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes192_gcm_by48_sgl_api_vaes_avx512.asm b/lib/avx512_t2/aes192_gcm_sgl_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes192_gcm_by48_sgl_api_vaes_avx512.asm rename to lib/avx512_t2/aes192_gcm_sgl_api_vaes_avx512.asm index f48c2aaa5986f2586842e417d2afdbaa141f1a18..0d1600bb1f9a1ba47303697f847c642a8b69884b 100644 --- a/lib/avx512_t2/aes192_gcm_by48_sgl_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes192_gcm_sgl_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512_t2/gcm_sgl_api_vaes_avx512.inc" +%include "include/gcm_sgl_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes192_gmac_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes192_gmac_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes192_gmac_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes192_gmac_api_vaes_avx512.asm index 71d785fd0a9bb5a2804d250bf358b977a90b5cd5..d82a5e20dc24f506030d22c3c5c0957bfec74ea0 100644 --- a/lib/avx512_t2/aes192_gmac_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes192_gmac_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512_t2/gcm_gmac_api_vaes_avx512.inc" +%include "include/gcm_gmac_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes256_gcm_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes256_gcm_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes256_gcm_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes256_gcm_api_vaes_avx512.asm index b302958cba4b6f490b651f9bdc4dce9efa256ec3..1e765b616c1ed42afcb3aafb4e29ddebb1ef03f2 100644 --- a/lib/avx512_t2/aes256_gcm_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes256_gcm_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512_t2/gcm_api_vaes_avx512.inc" +%include "include/gcm_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes256_gcm_by48_sgl_api_vaes_avx512.asm b/lib/avx512_t2/aes256_gcm_sgl_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes256_gcm_by48_sgl_api_vaes_avx512.asm rename to lib/avx512_t2/aes256_gcm_sgl_api_vaes_avx512.asm index bf705eb8f8d8f79d4cb1ebde31c5a34dc0bcb65d..dc73ee7998a2be4463fe32e6c0477ce05face8d9 100644 --- a/lib/avx512_t2/aes256_gcm_by48_sgl_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes256_gcm_sgl_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512_t2/gcm_sgl_api_vaes_avx512.inc" +%include "include/gcm_sgl_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes256_gmac_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes256_gmac_api_vaes_avx512.asm similarity index 93% rename from lib/avx512_t2/aes256_gmac_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes256_gmac_api_vaes_avx512.asm index bf648ffb2dd19138798b30bf47b16fe1516822c8..422e2ba44f363406ccb2b01b66224f23bb1a057e 100644 --- a/lib/avx512_t2/aes256_gmac_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes256_gmac_api_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512_t2/gcm_gmac_api_vaes_avx512.inc" +%include "include/gcm_gmac_api_vaes_avx512.inc" diff --git a/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm index 39ee837b2b512600e43c162f6719c513984a6f4b..df2cb80753c168c9059de449eca1e74260dfab57 100644 --- a/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/aes_common.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/aes_common.inc" +%include "include/clear_regs.inc" %define zIV zmm0 %define zBLK_0_3 zmm1 diff --git a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm index c916a83dd21597da5c6d6bbf1dbd3e54616c905f..3decdb2f2f3f8c0ae40e3cc9051f276cfaca34ef 100644 --- a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,10 +27,10 @@ ;;; routines to do 128/192/256 bit CBC AES encrypt -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" struc STACK _gpr_save: resq 4 diff --git a/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm index 2f5adea422e39b8252de9cbf130a7e2fe9e9b3fb..fed2ab86fcaa01e20cc85f76a151429480254179 100644 --- a/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm index 83a02d694bc75c38ea7eff7947f1703073a94061..1d617bc5d1454c2d9519118831d2f765822c4a2d 100644 --- a/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,10 +27,10 @@ ;;; routines to do 128 bit AES in CBCS mode encryption -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" %define GPR_SAVE_AREA rsp + _gpr_save diff --git a/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm index 62df455b3b18f623a9a28be1973d4e2cbcc2ba1b..4e75d7bc8bd719a63b8fe858f2e15c36145e7091 100644 --- a/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2019-2022, Intel Corporation All rights reserved. +; Copyright(c) 2019-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" +%include "include/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;void aes_cntr_128_submit_vaes_avx512 (IMB_JOB *job) diff --git a/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm index 431a8119d094e62533ba171e216e2affe562c52d..b03d69f3d97b89c2427cc138c12af1674c61250c 100644 --- a/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2019-2022, Intel Corporation All rights reserved. +; Copyright(c) 2019-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" +%include "include/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;void aes_cntr_bit_128_submit_vaes_avx512 (IMB_JOB *job) diff --git a/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm index 6dca75720ee143795d45608c6d0fb8a67ed72ed4..5e81d4f911e78d480d931ba607c5917e1535a7a2 100644 --- a/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2019-2022, Intel Corporation All rights reserved. +; Copyright(c) 2019-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" +%include "include/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;IMB_JOB * aes_cntr_ccm_128_vaes_avx512(IMB_JOB *job) diff --git a/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm index 74d59b4c6c32b4b4410595e044f7babc370f41d3..7352543e2dad71d09c335ad4e621ccc3189fa8c0 100644 --- a/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2019-2022, Intel Corporation All rights reserved. +; Copyright(c) 2019-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,7 +28,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" +%include "include/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;void aes_cntr_pon_enc_128_vaes_avx512 (void *src, void *dst, void *iv, void *keys, uint64_t length, uint32_t *bip) diff --git a/lib/avx512_t2/aes_docsis_dec_avx512.asm b/lib/avx512_t2/aes_docsis_dec_avx512.asm index f1dab4132c1e6ba15d91e85295fd353ada815f30..fb099c123b26eecc5b7225d678338600bca51655 100644 --- a/lib/avx512_t2/aes_docsis_dec_avx512.asm +++ b/lib/avx512_t2/aes_docsis_dec_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -26,11 +26,11 @@ ;; %use smartalign -%include "include/imb_job.asm" -%include "include/reg_sizes.asm" -%include "include/os.asm" -%include "include/clear_regs.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/imb_job.inc" +%include "include/reg_sizes.inc" +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/mb_mgr_datastruct.inc" ;; In System V AMD64 ABI ;; callee saves: RBX, RBP, R12-R15 diff --git a/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm b/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm index c03684aed734eb004e60c091a0502bbe930a1bee..ec1538e7e86305871488f4866a8d19ad3c63fb4b 100644 --- a/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm +++ b/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -26,12 +26,12 @@ ;; %use smartalign -%include "include/imb_job.asm" -%include "include/reg_sizes.asm" -%include "include/os.asm" -%include "include/clear_regs.asm" -%include "include/aes_common.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/imb_job.inc" +%include "include/reg_sizes.inc" +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" +%include "include/mb_mgr_datastruct.inc" default rel diff --git a/lib/avx512_t2/aes_docsis_enc_avx512.asm b/lib/avx512_t2/aes_docsis_enc_avx512.asm index 8a6a4e09097eeb5bf87757f95d5fec6a7ad00cd0..b6a3498465b22718d4d3e66d3fcefd8a2f0e1684 100644 --- a/lib/avx512_t2/aes_docsis_enc_avx512.asm +++ b/lib/avx512_t2/aes_docsis_enc_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -30,12 +30,12 @@ %use smartalign -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %define APPEND(a,b) a %+ b diff --git a/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm b/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm index fa7adb88e13ca3e507cd4a3ac303d635a8f15f3d..a10387c66c54fca922c333a8aedde2ff04f5e3db 100644 --- a/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -30,11 +30,11 @@ %use smartalign -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" %define APPEND(a,b) a %+ b diff --git a/lib/avx512_t2/aes_ecb_quic_vaes_avx512.asm b/lib/avx512_t2/aes_ecb_quic_vaes_avx512.asm new file mode 100644 index 0000000000000000000000000000000000000000..a322a95fde8de95c45b1d0006b43d5b83e847191 --- /dev/null +++ b/lib/avx512_t2/aes_ecb_quic_vaes_avx512.asm @@ -0,0 +1,294 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB encrypt for QUIC on 16-byte buffers + +; YMM registers are clobbered. Saving/restoring must be done at a higher level + +; void aes_ecb_quic_enc_x_vaes_avx512(void *in, +; UINT128 keys[], +; void *out, +; UINT64 num_buffers); +; +; x = key size (128/256) +; arg 1: IN: array of pointers to input buffers +; arg 2: KEYS: pointer to keys (common for all buffers) +; arg 3: OUT: array of pointers to output buffers) +; arg 4: N_BUFS: number of 16-byte buffers +; + +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" +%include "include/cet.inc" + +%define AES_ECB_QUIC_ENC_128 aes_ecb_quic_enc_128_vaes_avx512 +%define AES_ECB_QUIC_ENC_256 aes_ecb_quic_enc_256_vaes_avx512 + +%ifdef LINUX +%define IN rdi +%define KEYS rsi +%define OUT rdx +%define N_BUFS rcx +%else +%define IN rcx +%define KEYS rdx +%define OUT r8 +%define N_BUFS r9 +%endif +%define IDX rax +%define TMP r11 + +%define YKEY1 ymm1 +%define YDATA0 ymm2 +%define YDATA1 ymm3 +%define YDATA2 ymm4 +%define YDATA3 ymm5 +%define YDATA4 ymm6 +%define YDATA5 ymm7 +%define YDATA6 ymm8 +%define YDATA7 ymm9 + +mksection .text + +;; ============================================================================= +;; Loads specified single 16-byte block from different buffers into YMM registers +%macro YMM_LOAD_BLOCKS_MULT_IN_0_16 11 +%define %%NUM_BUFFERS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%ARRAY_INP %2 ; [in] array of input data pointers +%define %%INP %3 ; [clobbered] input data pointer to read 16 bytes from +%define %%DST0 %4 ; [out] YMM register with loaded data +%define %%DST1 %5 ; [out] YMM register with loaded data +%define %%DST2 %6 ; [out] YMM register with loaded data +%define %%DST3 %7 ; [out] YMM register with loaded data +%define %%DST4 %8 ; [out] YMM register with loaded data +%define %%DST5 %9 ; [out] YMM register with loaded data +%define %%DST6 %10 ; [out] YMM register with loaded data +%define %%DST7 %11 ; [out] YMM register with loaded data + +%assign dst_idx 0 +%assign buf_idx 0 + +%rep (%%NUM_BUFFERS / 2) +%xdefine %%DSTREG %%DST %+ dst_idx + mov %%INP, [%%ARRAY_INP + buf_idx] + vmovdqu8 XWORD(%%DSTREG), [%%INP] + mov %%INP, [%%ARRAY_INP + buf_idx + 8] + vinserti64x2 %%DSTREG, [%%INP], 1 +%undef %%DSTREG +%assign dst_idx (dst_idx + 1) +%assign buf_idx (buf_idx + 16) +%endrep + +%assign blocks_left (%%NUM_BUFFERS % 2) +%xdefine %%DSTREG %%DST %+ dst_idx + +%if blocks_left == 1 + mov %%INP, [%%ARRAY_INP + buf_idx] + vmovdqu8 XWORD(%%DSTREG), [%%INP] +%endif + +%endmacro + +;; ============================================================================= +;; Stores up to 16 bytes from YMM registers to different output buffers +%macro YMM_STORE_MASKED_BLOCKS_MULT_OUT_0_16 13 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%ARRAY_OUTP %2 ; [in] array of output data pointers to write to +%define %%OUTP %3 ; [clobbered] output data pointer to write to +%define %%SRC0 %4 ; [in] YMM register with data to store +%define %%SRC1 %5 ; [in] YMM register with data to store +%define %%SRC2 %6 ; [in] YMM register with data to store +%define %%SRC3 %7 ; [in] YMM register with data to store +%define %%SRC4 %8 ; [in] YMM register with data to store +%define %%SRC5 %9 ; [in] YMM register with data to store +%define %%SRC6 %10 ; [in] YMM register with data to store +%define %%SRC7 %11 ; [in] YMM register with data to store +%define %%XTMP %12 ; [clobbered] XMM register +%define %%KMASK %13 ; [in] K mask register + +%assign src_idx 0 +%assign buf_idx 0 + +%rep (%%NUM_BLOCKS / 2) +%xdefine %%SRCREG %%SRC %+ src_idx + mov %%OUTP, [%%ARRAY_OUTP + buf_idx] + vmovdqu8 [%%OUTP]{%%KMASK}, XWORD(%%SRCREG) + mov %%OUTP, [%%ARRAY_OUTP + buf_idx + 8] + vextracti64x2 %%XTMP, %%SRCREG, 1 + vmovdqu8 [%%OUTP]{%%KMASK}, %%XTMP +%undef %%SRCREG +%assign src_idx (src_idx + 1) +%assign buf_idx (buf_idx + 16) +%endrep + +%assign blocks_left (%%NUM_BLOCKS % 2) +%xdefine %%SRCREG %%SRC %+ src_idx + +%if blocks_left == 1 + mov %%OUTP, [%%ARRAY_OUTP + buf_idx] + vmovdqu8 [%%OUTP]{%%KMASK}, XWORD(%%SRCREG) +%endif + +%endmacro + +; +; Performs AES-ECB on 16-byte blocks from multiple buffers (IN, number = N_BUFS) +; and outputs 5 bytes of ciphertext to the same number of buffers (OUT), +; all sharing the same AES key +%macro AES_ECB_QUIC 1 +%define %%NROUNDS %1 ; [in] Number of AES rounds, numerical value + + or N_BUFS, N_BUFS + mov TMP, N_BUFS + jz %%done + + xor IDX, IDX + and TMP, 0xf + jz %%main_loop + + ; branch to different code block based on remainder + cmp TMP, 8 + je %%initial_num_buffers_is_8 + jb %%initial_num_buffers_is_7_1 + cmp TMP, 12 + je %%initial_num_buffers_is_12 + jb %%initial_num_buffers_is_11_9 + ;; 15, 14 or 13 + cmp TMP, 14 + ja %%initial_num_buffers_is_15 + je %%initial_num_buffers_is_14 + jmp %%initial_num_buffers_is_13 +%%initial_num_buffers_is_11_9: + ;; 11, 10 or 9 + cmp TMP, 10 + ja %%initial_num_buffers_is_11 + je %%initial_num_buffers_is_10 + jmp %%initial_num_buffers_is_9 +%%initial_num_buffers_is_7_1: + cmp TMP, 4 + je %%initial_num_buffers_is_4 + jb %%initial_num_buffers_is_3_1 + ;; 7, 6 or 5 + cmp TMP, 6 + ja %%initial_num_buffers_is_7 + je %%initial_num_buffers_is_6 + jmp %%initial_num_buffers_is_5 +%%initial_num_buffers_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2 + ja %%initial_num_buffers_is_3 + je %%initial_num_buffers_is_2 + ;; fall through for `jmp %%initial_num_buffers_is_1` + +%assign num_buffers 1 +%rep 15 + + %%initial_num_buffers_is_ %+ num_buffers : +%assign %%I 0 + ; load blocks + YMM_LOAD_BLOCKS_MULT_IN_0_16 num_buffers, IN, TMP, YDATA0,\ + YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7 + +; Perform AES encryption on blocks +%rep (%%NROUNDS + 1) ; 10/14 + vbroadcasti128 YKEY1, [KEYS + %%I*16] + YMM_AESENC_ROUND_BLOCKS_0_16 YDATA0, YDATA1, YDATA2, YDATA3, YDATA4,\ + YDATA5, YDATA6, YDATA7, YKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, num_buffers, (%%NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + + ; store blocks + mov TMP, 0x1f + kmovq k1, TMP + YMM_STORE_MASKED_BLOCKS_MULT_OUT_0_16 num_buffers, OUT, TMP, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7, XWORD(YKEY1), k1 + + add IDX, num_buffers + cmp IDX, N_BUFS + je %%done + +%assign num_buffers (num_buffers + 1) + jmp %%main_loop +%endrep + +align 16 +%%main_loop: + ; load next 16 blocks + YMM_LOAD_BLOCKS_MULT_IN_0_16 16, {IN + IDX*8}, TMP, YDATA0,\ + YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7 + + ; Perform AES encryption on 16 blocks +%assign %%ROUNDNO 0 ; current key number +%rep (%%NROUNDS + 1) ; 10/14 + vbroadcasti128 YKEY1, [KEYS + %%ROUNDNO*16] + YMM_AESENC_ROUND_BLOCKS_0_16 YDATA0, YDATA1, YDATA2, YDATA3, YDATA4,\ + YDATA5, YDATA6, YDATA7, YKEY1, %%ROUNDNO, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, 16, (%%NROUNDS - 1) + +%assign %%ROUNDNO (%%ROUNDNO + 1) +%endrep + + ; write 16 blocks to output + mov TMP, 0x1f + kmovq k1, TMP + YMM_STORE_MASKED_BLOCKS_MULT_OUT_0_16 16, {OUT + IDX*8}, TMP, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7, XWORD(YKEY1), k1 + + add IDX, 16 + + cmp IDX, N_BUFS + jne %%main_loop + +%%done: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif +%endmacro + +align 16 +MKGLOBAL(AES_ECB_QUIC_ENC_128,function,internal) +AES_ECB_QUIC_ENC_128: + endbranch64 + AES_ECB_QUIC 10 + ret + +align 16 +MKGLOBAL(AES_ECB_QUIC_ENC_256,function,internal) +AES_ECB_QUIC_ENC_256: + endbranch64 + AES_ECB_QUIC 14 + ret + +mksection stack-noexec diff --git a/lib/avx512_t2/aes_ecb_vaes_avx512.asm b/lib/avx512_t2/aes_ecb_vaes_avx512.asm index a4287a302ff4f51d33b892f2bc75d733167c568e..3c2fe3f3768b510e927f20f0a0ea2d826578f3c1 100644 --- a/lib/avx512_t2/aes_ecb_vaes_avx512.asm +++ b/lib/avx512_t2/aes_ecb_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -42,9 +42,9 @@ ; arg 4: LEN: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" -%include "include/aes_common.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" %define AES_ECB_ENC_128 aes_ecb_enc_128_vaes_avx512 %define AES_ECB_DEC_128 aes_ecb_dec_128_vaes_avx512 diff --git a/lib/avx512_t2/crc16_x25_avx512.asm b/lib/avx512_t2/crc16_x25_avx512.asm index 407665eca67f62f6b62508439cdceb870504fa8b..bc4cf6eb5dc26c495644cc4c37d7763438efaf24 100644 --- a/lib/avx512_t2/crc16_x25_avx512.asm +++ b/lib/avx512_t2/crc16_x25_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" %include "include/crc32_refl_const.inc" %include "include/crc32_refl.inc" %include "include/cet.inc" diff --git a/lib/avx512_t2/crc32_by16_vclmul_avx512.asm b/lib/avx512_t2/crc32_by16_vclmul_avx512.asm index 03e38cc34922facaff68f95e0b9e402afc45fcba..70288ac19d157f8515d7c9cc8ece4e801d1be57e 100644 --- a/lib/avx512_t2/crc32_by16_vclmul_avx512.asm +++ b/lib/avx512_t2/crc32_by16_vclmul_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -35,12 +35,12 @@ ;; "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" ;; URL: http://download.intel.com/design/intarch/papers/323102.pdf -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" %include "include/crc32.inc" %include "include/cet.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" [bits 64] default rel diff --git a/lib/avx512_t2/crc32_fp_avx512.asm b/lib/avx512_t2/crc32_fp_avx512.asm index 6e64d904946890f69281117a79ad85284d5b2154..09ab985cda227f41268a17829f80b205648e7c98 100644 --- a/lib/avx512_t2/crc32_fp_avx512.asm +++ b/lib/avx512_t2/crc32_fp_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx512_t2/crc32_iuup_avx512.asm b/lib/avx512_t2/crc32_iuup_avx512.asm index 6531f11efb9caebe06b3723f7039b556bb3f1248..e364732417348118bccff4f7f8fe47b65c37530c 100644 --- a/lib/avx512_t2/crc32_iuup_avx512.asm +++ b/lib/avx512_t2/crc32_iuup_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx512_t2/crc32_lte_avx512.asm b/lib/avx512_t2/crc32_lte_avx512.asm index 77a4373e9b0a084fcbdb69ff80c235b7dd05f3e8..683152a43fe226af6f8cc3cd7eff294999ba5adc 100644 --- a/lib/avx512_t2/crc32_lte_avx512.asm +++ b/lib/avx512_t2/crc32_lte_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm b/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm index 6b9a735b9825e986415bcbbf36e50707107d2b5e..67dfb64517a00af4b3e9a202b7b25291d405b7a9 100644 --- a/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm +++ b/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -35,12 +35,12 @@ ;; "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" ;; URL: http://download.intel.com/design/intarch/papers/323102.pdf -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" %include "include/crc32_refl.inc" %include "include/cet.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" [bits 64] default rel diff --git a/lib/avx512_t2/crc32_sctp_avx512.asm b/lib/avx512_t2/crc32_sctp_avx512.asm index 9996000bd597c55e97df24edeec59236d193eaa7..2e1d3acdbb073cd3cfb04fb0831745950be61328 100644 --- a/lib/avx512_t2/crc32_sctp_avx512.asm +++ b/lib/avx512_t2/crc32_sctp_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx512_t2/crc32_wimax_avx512.asm b/lib/avx512_t2/crc32_wimax_avx512.asm index 13a7288e25a04408f99c0c8e3b4fc5001b03ef5b..79a14cf2dbc2fc2509bb48dd75187129e981289b 100644 --- a/lib/avx512_t2/crc32_wimax_avx512.asm +++ b/lib/avx512_t2/crc32_wimax_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx512_t2/ethernet_fcs_avx512.asm b/lib/avx512_t2/ethernet_fcs_avx512.asm index 64d9a4e551234a6eb809de8016e6299b59b3bf8d..80574de0a07ffad29af53d0a5ee5889796294b5a 100644 --- a/lib/avx512_t2/ethernet_fcs_avx512.asm +++ b/lib/avx512_t2/ethernet_fcs_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" %include "include/crc32_refl_const.inc" %include "include/crc32_refl.inc" %include "include/cet.inc" diff --git a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm index 90bfd10407d0e3e02fb6f114e6542ce5c07c7772..6b7e28bf5036040346bd334aeface22c72f36062 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/constants.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/constants.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_ENC_X16 %define AES_CBC_ENC_X16 aes_cbc_enc_128_flush_vaes_avx512 diff --git a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm index 2acf75f7582c1694a2603dbc3eebd8b37dfc3768..291d916c04989f0653ea57b00d3405e2f4e4c5d7 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifndef AES_CBC_ENC_X16 %define AES_CBC_ENC_X16 aes_cbc_enc_128_vaes_avx512 diff --git a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm index f7bb94f6e81cf8d58ec6a3b3ec379d4056098329..0ea4b15301201eeecdab1a713004cb14b68a8969 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm index 6497f60fc2cc96d8c4361234a85847023019a16f..91a6a768e78154516d67e668aaa07d4fe7d8833f 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm index bc5be4525cc34f83d9f6d38ea065a227f6d1e33b..f04a6d81342829fb0879d40c08406ee9bfff3f40 100644 --- a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,13 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_MAC %define AES_CBC_MAC aes128_cbc_mac_vaes_avx512 diff --git a/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm index 9a1acc8a84ea12f9e6fe5571f0aa01b64c5332de..3fe23b678681fb463d68017360d42cf607500fbd 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifndef AES_CBC_MAC %define AES_CBC_MAC aes128_cbc_mac_vaes_avx512 diff --git a/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm index ee900051ea17ae3dbf56122365700e940acd72d9..0ae7f2a0c4b4dded35f24dc0041a1c2ffa71895c 100644 --- a/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,13 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifndef AES_XCBC_X16 %define AES_XCBC_X16 aes_xcbc_mac_128_vaes_avx512 diff --git a/lib/avx512_t2/mb_mgr_aes192_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_flush_avx512.asm index 8a0416c940a4c9cef119feb9c6b9d759da51c914..c254dd3473edb58f0d9cbc36fb6cb4f2e6e0bc74 100644 --- a/lib/avx512_t2/mb_mgr_aes192_cbc_enc_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_aes192_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_submit_avx512.asm index 1bf43590b9c2ef67187d05db71876b13705cf187..0bc55abb09e51a7c6123f746812500c42bc6edde 100644 --- a/lib/avx512_t2/mb_mgr_aes192_cbc_enc_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_aes256_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_flush_avx512.asm index a3ec6b0cc48bbcc48092e973c793cc35c3123af1..a6acbcbe30660a468d0f866f6f49c80cf9398116 100644 --- a/lib/avx512_t2/mb_mgr_aes256_cbc_enc_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_flush_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_aes256_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_submit_avx512.asm index 3e001baf8c712e1bc46160b8a7f53238a76848f8..7f7b0f396b129dfad9b96d22ea61793f73a1bd54 100644 --- a/lib/avx512_t2/mb_mgr_aes256_cbc_enc_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_submit_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm index 6bfdf6ea539c50821c071a6a16400fdf0dc9dc96..a16a41d4f7dd78e57c5c241dba0e8309ca34fd90 100644 --- a/lib/avx512_t2/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm index d0224cf06849f73e278dd57230b717603a851506..e3fe10da8c23fe4b27d0e2eb382cd88f97e3c2a9 100644 --- a/lib/avx512_t2/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/mb_mgr_avx512_t2.c b/lib/avx512_t2/mb_mgr_avx512_t2.c index 5447b79714ce7974fc9f46a3bf155d413e8ee1bb..3d2e5fb63c92ccae686fcdafbe5b6b6f4a2370b8 100644 --- a/lib/avx512_t2/mb_mgr_avx512_t2.c +++ b/lib/avx512_t2/mb_mgr_avx512_t2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -48,6 +48,7 @@ #include "include/aesni_emu.h" #include "include/error.h" +#include "include/arch_sse_type2.h" /* SHA-NI */ #include "include/arch_avx_type1.h" /* AESNI */ #include "include/arch_avx2_type1.h" /* MD5 */ #include "include/arch_avx512_type1.h" @@ -79,6 +80,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512_t2 #define SUBMIT_HASH_BURST submit_hash_burst_avx512_t2 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx512_t2 +#define SET_SUITE_ID_FN set_suite_id_avx512_t2 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512_T2 @@ -118,10 +120,13 @@ #define AES_CBC_DEC_256 aes_cbc_dec_256_vaes_avx512 /* AES-CBCS */ -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_vaes_avx512 -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_vaes_avx512 -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_vaes_avx512 -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_vaes_avx512 +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC \ + submit_job_aes128_cbcs_1_9_enc_vaes_avx512 +#define FLUSH_JOB_AES128_CBCS_1_9_ENC \ + flush_job_aes128_cbcs_1_9_enc_vaes_avx512 +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC \ + submit_job_aes128_cbcs_1_9_dec_vaes_avx512 +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_vaes_avx512 /* AES-ECB */ #define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx512 @@ -339,12 +344,18 @@ submit_job_docsis256_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, return job; } -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC submit_job_aes_docsis128_enc_crc32_vaes_avx512 -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC submit_job_aes_docsis256_enc_crc32_vaes_avx512 -#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC flush_job_aes_docsis128_enc_crc32_vaes_avx512 -#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC flush_job_aes_docsis256_enc_crc32_vaes_avx512 -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC submit_job_docsis128_sec_crc_dec_vaes_avx512 -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC submit_job_docsis256_sec_crc_dec_vaes_avx512 +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC \ + submit_job_aes_docsis128_enc_crc32_vaes_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC \ + submit_job_aes_docsis256_enc_crc32_vaes_avx512 +#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC \ + flush_job_aes_docsis128_enc_crc32_vaes_avx512 +#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC \ + flush_job_aes_docsis256_enc_crc32_vaes_avx512 +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC \ + submit_job_docsis128_sec_crc_dec_vaes_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC \ + submit_job_docsis256_sec_crc_dec_vaes_avx512 /* ====================================================================== */ @@ -470,6 +481,7 @@ init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_avx512; state->keyexp_192 = aes_keyexp_192_avx512; @@ -482,12 +494,12 @@ init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs) state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx512; - state->sha1 = sha1_avx512; - state->sha224_one_block = sha224_one_block_avx512; - state->sha224 = sha224_avx512; - state->sha256_one_block = sha256_one_block_avx512; - state->sha256 = sha256_avx512; + state->sha1_one_block = sha1_one_block_sse_shani; + state->sha1 = sha1_sse_shani; + state->sha224_one_block = sha224_one_block_sse_shani; + state->sha224 = sha224_sse_shani; + state->sha256_one_block = sha256_one_block_sse_shani; + state->sha256 = sha256_sse_shani; state->sha384_one_block = sha384_one_block_avx512; state->sha384 = sha384_avx512; state->sha512_one_block = sha512_one_block_avx512; @@ -495,6 +507,7 @@ init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_avx512; state->aes128_cfb_one = aes_cfb_128_one_avx512; + state->aes256_cfb_one = aes_cfb_256_one_avx512; state->eea3_1_buffer = zuc_eea3_1_buffer_avx512; state->eea3_4_buffer = zuc_eea3_4_buffer_avx; @@ -597,6 +610,9 @@ init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_vaes_avx512; state->gmac192_finalize = imb_aes_gmac_finalize_192_vaes_avx512; state->gmac256_finalize = imb_aes_gmac_finalize_256_vaes_avx512; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_vaes_avx512; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_vaes_avx512; } #include "mb_mgr_code.h" diff --git a/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm index 380446c54725a817d17164ba0c944cb490898836..c6682477e0f22b4d728809877eef79511b2f10bd 100644 --- a/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,14 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/mb_mgr_datastruct.asm" -%include "include/datastruct.asm" -%include "include/transpose_avx512.asm" -%include "include/imb_job.asm" -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/mb_mgr_datastruct.inc" +%include "include/datastruct.inc" +%include "include/transpose_avx512.inc" +%include "include/imb_job.inc" +%include "include/os.inc" +%include "include/clear_regs.inc" -%include "avx512_t2/snow3g_uea2_by16_vaes_avx512.asm" +%include "include/snow3g_uea2_by16_vaes_avx512.inc" %ifndef SUBMIT_JOB_SNOW3G_UEA2 %define SUBMIT_JOB_SNOW3G_UEA2_GEN2 submit_job_snow3g_uea2_vaes_avx512 diff --git a/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm index c10985b1e49f26f5004d668861a1f0446085227c..d4043388b3354377416f614f126161f98b26b813 100644 --- a/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,14 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/constants.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/constants.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" -%include "avx512_t2/snow3g_uea2_by16_vaes_avx512.asm" +%include "include/clear_regs.inc" +%include "include/snow3g_uea2_by16_vaes_avx512.inc" %ifndef SUBMIT_JOB_SNOW3G_UIA2 %define SUBMIT_JOB_SNOW3G_UIA2_GEN2 submit_job_snow3g_uia2_vaes_avx512 diff --git a/lib/avx512_t2/mb_mgr_zuc_submit_flush_gfni_avx512.asm b/lib/avx512_t2/mb_mgr_zuc_submit_flush_gfni_avx512.asm index 76649d36557185118e4c31e5901416a5caa5149e..abf8acf045b00d2d8105ed29ad56955e417f3f41 100644 --- a/lib/avx512_t2/mb_mgr_zuc_submit_flush_gfni_avx512.asm +++ b/lib/avx512_t2/mb_mgr_zuc_submit_flush_gfni_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx512_t2/poly_fma_avx512.asm b/lib/avx512_t2/poly_fma_avx512.asm index 0eec89f38b1d7e6203800db41f29ef3160cb24d4..3e04eef81a6b084a03a49bd5afb142d06446716c 100644 --- a/lib/avx512_t2/poly_fma_avx512.asm +++ b/lib/avx512_t2/poly_fma_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" -%include "include/imb_job.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" ;; Enforce EVEX encoding for AVX512 capable systems @@ -666,16 +666,16 @@ mksection .text %define %%B0 %4 ; [in/out] ZMM register containing 1st 44-bit limb of the 8 blocks %define %%B1 %5 ; [in/out] ZMM register containing 2nd 44-bit limb of the 8 blocks %define %%B2 %6 ; [in/out] ZMM register containing 3rd 44-bit limb of the 8 blocks -%define %%R0 %7 ; [in] ZMM register (R0) to include the 1st limb in IDX -%define %%R1 %8 ; [in] ZMM register (R1) to include the 2nd limb in IDX -%define %%R2 %9 ; [in] ZMM register (R2) to include the 3rd limb in IDX -%define %%R1P %10 ; [in] ZMM register (R1') to include the 2nd limb (multiplied by 5) in IDX -%define %%R2P %11 ; [in] ZMM register (R2') to include the 3rd limb (multiplied by 5) in IDX -%define %%S0 %12 ; [in] ZMM register (R0) to include the 1st limb in IDX -%define %%S1 %13 ; [in] ZMM register (R1) to include the 2nd limb in IDX -%define %%S2 %14 ; [in] ZMM register (R2) to include the 3rd limb in IDX -%define %%S1P %15 ; [in] ZMM register (R1') to include the 2nd limb (multiplied by 5) in IDX -%define %%S2P %16 ; [in] ZMM register (R2') to include the 3rd limb (multiplied by 5) in IDX +%define %%R0 %7 ; [in] ZMM register (R0) to include the 1st limb in R +%define %%R1 %8 ; [in] ZMM register (R1) to include the 2nd limb in R +%define %%R2 %9 ; [in] ZMM register (R2) to include the 3rd limb in R +%define %%R1P %10 ; [in] ZMM register (R1') to include the 2nd limb (multiplied by 5) in R +%define %%R2P %11 ; [in] ZMM register (R2') to include the 3rd limb (multiplied by 5) in R +%define %%S0 %12 ; [in] ZMM register (S0) to include the 1st limb in S +%define %%S1 %13 ; [in] ZMM register (S1) to include the 2nd limb in S +%define %%S2 %14 ; [in] ZMM register (S2) to include the 3rd limb in S +%define %%S1P %15 ; [in] ZMM register (S1') to include the 2nd limb (multiplied by 5) in S +%define %%S2P %16 ; [in] ZMM register (S2') to include the 3rd limb (multiplied by 5) in S %define %%P0_L %17 ; [clobbered] ZMM register to contain p[0] of the 8 blocks %define %%P0_H %18 ; [clobbered] ZMM register to contain p[0] of the 8 blocks %define %%P1_L %19 ; [clobbered] ZMM register to contain p[1] of the 8 blocks diff --git a/lib/avx512_t2/pon_vaes_avx512.asm b/lib/avx512_t2/pon_vaes_avx512.asm index 1f712f15b6429329a35349dd1a5bc76e8e67f86c..895c6dc9eca4f4b0861918e0c8cadc6758b96b78 100644 --- a/lib/avx512_t2/pon_vaes_avx512.asm +++ b/lib/avx512_t2/pon_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,10 +27,10 @@ %use smartalign -%include "include/imb_job.asm" -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" +%include "include/imb_job.inc" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" extern aes_cntr_pon_enc_128_vaes_avx512 extern aes_cntr_pon_dec_128_vaes_avx512 diff --git a/lib/avx512_t2/snow3g_uia2_by32_vaes_avx512.asm b/lib/avx512_t2/snow3g_uia2_by32_vaes_avx512.asm index e9e9a3e3fab9adfa1af97b330ce9962b8dc076a9..a815c7e6c04d19cd05c7c0ab638f93b0beb6a7fc 100644 --- a/lib/avx512_t2/snow3g_uia2_by32_vaes_avx512.asm +++ b/lib/avx512_t2/snow3g_uia2_by32_vaes_avx512.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/cet.inc" -%include "include/memcpy.asm" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/aes_common.asm" +%include "include/aes_common.inc" %define APPEND(a,b) a %+ b %define APPEND3(a,b,c) a %+ b %+ c diff --git a/lib/avx512_t2/zuc_x16_vaes_avx512.asm b/lib/avx512_t2/zuc_x16_vaes_avx512.asm index 42446d4f3547e1471c282f3136a4780a8be8320d..98ad69b01551b340b2fd55521d15391132816482 100644 --- a/lib/avx512_t2/zuc_x16_vaes_avx512.asm +++ b/lib/avx512_t2/zuc_x16_vaes_avx512.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/avx_t1/aes128_cbc_dec_by8_avx.asm b/lib/avx_t1/aes128_cbc_dec_by8_avx.asm index c50d7898d30fdafd16268601b4a074fe72686ef6..93ab4b06c524ee2496eaa32ebfda96d2f380867d 100644 --- a/lib/avx_t1/aes128_cbc_dec_by8_avx.asm +++ b/lib/avx_t1/aes128_cbc_dec_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,8 +29,8 @@ ;; clobbers xmm0-15 -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu diff --git a/lib/avx_t1/aes128_cbc_enc_x8_avx.asm b/lib/avx_t1/aes128_cbc_enc_x8_avx.asm index 3ae67c9b40fe578c2e1aabfb478bfbe1cb194735..516eb0de2185011150c3f72608d4fddeddb1e5c3 100644 --- a/lib/avx_t1/aes128_cbc_enc_x8_avx.asm +++ b/lib/avx_t1/aes128_cbc_enc_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,499 +25,16 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -;;; routine to do a 128 bit CBC AES encrypt and CBC MAC +;; AES-CBC-128 -;; clobbers all registers except for ARG1 and rbp - -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" - -%define VMOVDQ vmovdqu ;; assume buffers not aligned - -%macro VPXOR2 2 - vpxor %1, %1, %2 -%endm - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; struct AES_ARGS { -;; void* in[8]; -;; void* out[8]; -;; UINT128* keys[8]; -;; UINT128 IV[8]; -;; } -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; void aes_cbc_enc_128_x8(AES_ARGS *args, UINT64 len); -;; arg 1: ARG : addr of AES_ARGS structure -;; arg 2: LEN : len (in units of bytes) - -struc STACK -_gpr_save: resq 8 -_len: resq 1 -endstruc - -%define GPR_SAVE_AREA rsp + _gpr_save -%define LEN_AREA rsp + _len - -%ifdef LINUX -%define arg1 rdi -%define arg2 rsi -%define arg3 rcx -%define arg4 rdx -%else -%define arg1 rcx -%define arg2 rdx -%define arg3 rdi -%define arg4 rsi -%endif - -%define ARG arg1 -%define LEN arg2 - -%define IDX rax -%define TMP rbx - -%define KEYS0 arg3 -%define KEYS1 arg4 -%define KEYS2 rbp -%define KEYS3 r8 -%define KEYS4 r9 -%define KEYS5 r10 -%define KEYS6 r11 -%define KEYS7 r12 - -%define IN0 r13 -%define IN2 r14 -%define IN4 r15 -%define IN6 LEN - -%define XDATA0 xmm0 -%define XDATA1 xmm1 -%define XDATA2 xmm2 -%define XDATA3 xmm3 -%define XDATA4 xmm4 -%define XDATA5 xmm5 -%define XDATA6 xmm6 -%define XDATA7 xmm7 - -%define XKEY0_3 xmm8 -%define XKEY1_4 xmm9 -%define XKEY2_5 xmm10 -%define XKEY3_6 xmm11 -%define XKEY4_7 xmm12 -%define XKEY5_8 xmm13 -%define XKEY6_9 xmm14 -%define XTMP xmm15 +%include "include/aes_cbc_enc_x8_avx.inc" mksection .text -%macro AES_CBC_X8 5-6 -%define %%MODE %1 -%define %%OFFSET %2 -%define %%ARG_IV %3 -%define %%ARG_KEYS %4 -%define %%ARG_IN %5 -%define %%ARG_OUT %6 - - sub rsp, STACK_size - mov [GPR_SAVE_AREA + 8*0], rbp -%ifidn %%MODE, CBC_XCBC_MAC - mov [GPR_SAVE_AREA + 8*1], rbx - mov [GPR_SAVE_AREA + 8*2], r12 - mov [GPR_SAVE_AREA + 8*3], r13 - mov [GPR_SAVE_AREA + 8*4], r14 - mov [GPR_SAVE_AREA + 8*5], r15 -%ifndef LINUX - mov [GPR_SAVE_AREA + 8*6], rsi - mov [GPR_SAVE_AREA + 8*7], rdi -%endif -%endif - - mov IDX, %%OFFSET - mov [LEN_AREA], LEN - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - mov IN0, [ARG + %%ARG_IN + 8*0] - mov IN2, [ARG + %%ARG_IN + 8*2] - mov IN4, [ARG + %%ARG_IN + 8*4] - mov IN6, [ARG + %%ARG_IN + 8*6] - - mov TMP, [ARG + %%ARG_IN + 8*1] - VMOVDQ XDATA0, [IN0] ; load first block of plain text - VMOVDQ XDATA1, [TMP] ; load first block of plain text - mov TMP, [ARG + %%ARG_IN + 8*3] - VMOVDQ XDATA2, [IN2] ; load first block of plain text - VMOVDQ XDATA3, [TMP] ; load first block of plain text - mov TMP, [ARG + %%ARG_IN + 8*5] - VMOVDQ XDATA4, [IN4] ; load first block of plain text - VMOVDQ XDATA5, [TMP] ; load first block of plain text - mov TMP, [ARG + %%ARG_IN + 8*7] - VMOVDQ XDATA6, [IN6] ; load first block of plain text - VMOVDQ XDATA7, [TMP] ; load first block of plain text - - VPXOR2 XDATA0, [ARG + %%ARG_IV + 16*0] ; plaintext XOR IV - VPXOR2 XDATA1, [ARG + %%ARG_IV + 16*1] ; plaintext XOR IV - VPXOR2 XDATA2, [ARG + %%ARG_IV + 16*2] ; plaintext XOR IV - VPXOR2 XDATA3, [ARG + %%ARG_IV + 16*3] ; plaintext XOR IV - VPXOR2 XDATA4, [ARG + %%ARG_IV + 16*4] ; plaintext XOR IV - VPXOR2 XDATA5, [ARG + %%ARG_IV + 16*5] ; plaintext XOR IV - VPXOR2 XDATA6, [ARG + %%ARG_IV + 16*6] ; plaintext XOR IV - VPXOR2 XDATA7, [ARG + %%ARG_IV + 16*7] ; plaintext XOR IV - - mov KEYS0, [ARG + %%ARG_KEYS + 8*0] - mov KEYS1, [ARG + %%ARG_KEYS + 8*1] - mov KEYS2, [ARG + %%ARG_KEYS + 8*2] - mov KEYS3, [ARG + %%ARG_KEYS + 8*3] - mov KEYS4, [ARG + %%ARG_KEYS + 8*4] - mov KEYS5, [ARG + %%ARG_KEYS + 8*5] - mov KEYS6, [ARG + %%ARG_KEYS + 8*6] - mov KEYS7, [ARG + %%ARG_KEYS + 8*7] - - VPXOR2 XDATA0, [KEYS0 + 16*0] ; 0. ARK - VPXOR2 XDATA1, [KEYS1 + 16*0] ; 0. ARK - VPXOR2 XDATA2, [KEYS2 + 16*0] ; 0. ARK - VPXOR2 XDATA3, [KEYS3 + 16*0] ; 0. ARK - VPXOR2 XDATA4, [KEYS4 + 16*0] ; 0. ARK - VPXOR2 XDATA5, [KEYS5 + 16*0] ; 0. ARK - VPXOR2 XDATA6, [KEYS6 + 16*0] ; 0. ARK - VPXOR2 XDATA7, [KEYS7 + 16*0] ; 0. ARK - - vaesenc XDATA0, [KEYS0 + 16*1] ; 1. ENC - vaesenc XDATA1, [KEYS1 + 16*1] ; 1. ENC - vaesenc XDATA2, [KEYS2 + 16*1] ; 1. ENC - vaesenc XDATA3, [KEYS3 + 16*1] ; 1. ENC - vaesenc XDATA4, [KEYS4 + 16*1] ; 1. ENC - vaesenc XDATA5, [KEYS5 + 16*1] ; 1. ENC - vaesenc XDATA6, [KEYS6 + 16*1] ; 1. ENC - vaesenc XDATA7, [KEYS7 + 16*1] ; 1. ENC - - vmovdqa XKEY0_3, [KEYS0 + 16*3] ; load round 3 key - - vaesenc XDATA0, [KEYS0 + 16*2] ; 2. ENC - vaesenc XDATA1, [KEYS1 + 16*2] ; 2. ENC - vaesenc XDATA2, [KEYS2 + 16*2] ; 2. ENC - vaesenc XDATA3, [KEYS3 + 16*2] ; 2. ENC - vaesenc XDATA4, [KEYS4 + 16*2] ; 2. ENC - vaesenc XDATA5, [KEYS5 + 16*2] ; 2. ENC - vaesenc XDATA6, [KEYS6 + 16*2] ; 2. ENC - vaesenc XDATA7, [KEYS7 + 16*2] ; 2. ENC - - vmovdqa XKEY1_4, [KEYS1 + 16*4] ; load round 4 key - - vaesenc XDATA0, XKEY0_3 ; 3. ENC - vaesenc XDATA1, [KEYS1 + 16*3] ; 3. ENC - vaesenc XDATA2, [KEYS2 + 16*3] ; 3. ENC - vaesenc XDATA3, [KEYS3 + 16*3] ; 3. ENC - vaesenc XDATA4, [KEYS4 + 16*3] ; 3. ENC - vaesenc XDATA5, [KEYS5 + 16*3] ; 3. ENC - vaesenc XDATA6, [KEYS6 + 16*3] ; 3. ENC - vaesenc XDATA7, [KEYS7 + 16*3] ; 3. ENC - - vaesenc XDATA0, [KEYS0 + 16*4] ; 4. ENC - vmovdqa XKEY2_5, [KEYS2 + 16*5] ; load round 5 key - vaesenc XDATA1, XKEY1_4 ; 4. ENC - vaesenc XDATA2, [KEYS2 + 16*4] ; 4. ENC - vaesenc XDATA3, [KEYS3 + 16*4] ; 4. ENC - vaesenc XDATA4, [KEYS4 + 16*4] ; 4. ENC - vaesenc XDATA5, [KEYS5 + 16*4] ; 4. ENC - vaesenc XDATA6, [KEYS6 + 16*4] ; 4. ENC - vaesenc XDATA7, [KEYS7 + 16*4] ; 4. ENC - - vaesenc XDATA0, [KEYS0 + 16*5] ; 5. ENC - vaesenc XDATA1, [KEYS1 + 16*5] ; 5. ENC - vmovdqa XKEY3_6, [KEYS3 + 16*6] ; load round 6 key - vaesenc XDATA2, XKEY2_5 ; 5. ENC - vaesenc XDATA3, [KEYS3 + 16*5] ; 5. ENC - vaesenc XDATA4, [KEYS4 + 16*5] ; 5. ENC - vaesenc XDATA5, [KEYS5 + 16*5] ; 5. ENC - vaesenc XDATA6, [KEYS6 + 16*5] ; 5. ENC - vaesenc XDATA7, [KEYS7 + 16*5] ; 5. ENC - - vaesenc XDATA0, [KEYS0 + 16*6] ; 6. ENC - vaesenc XDATA1, [KEYS1 + 16*6] ; 6. ENC - vaesenc XDATA2, [KEYS2 + 16*6] ; 6. ENC - vmovdqa XKEY4_7, [KEYS4 + 16*7] ; load round 7 key - vaesenc XDATA3, XKEY3_6 ; 6. ENC - vaesenc XDATA4, [KEYS4 + 16*6] ; 6. ENC - vaesenc XDATA5, [KEYS5 + 16*6] ; 6. ENC - vaesenc XDATA6, [KEYS6 + 16*6] ; 6. ENC - vaesenc XDATA7, [KEYS7 + 16*6] ; 6. ENC - - vaesenc XDATA0, [KEYS0 + 16*7] ; 7. ENC - vaesenc XDATA1, [KEYS1 + 16*7] ; 7. ENC - vaesenc XDATA2, [KEYS2 + 16*7] ; 7. ENC - vaesenc XDATA3, [KEYS3 + 16*7] ; 7. ENC - vmovdqa XKEY5_8, [KEYS5 + 16*8] ; load round 8 key - vaesenc XDATA4, XKEY4_7 ; 7. ENC - vaesenc XDATA5, [KEYS5 + 16*7] ; 7. ENC - vaesenc XDATA6, [KEYS6 + 16*7] ; 7. ENC - vaesenc XDATA7, [KEYS7 + 16*7] ; 7. ENC - - vaesenc XDATA0, [KEYS0 + 16*8] ; 8. ENC - vaesenc XDATA1, [KEYS1 + 16*8] ; 8. ENC - vaesenc XDATA2, [KEYS2 + 16*8] ; 8. ENC - vaesenc XDATA3, [KEYS3 + 16*8] ; 8. ENC - vaesenc XDATA4, [KEYS4 + 16*8] ; 8. ENC - vmovdqa XKEY6_9, [KEYS6 + 16*9] ; load round 9 key - vaesenc XDATA5, XKEY5_8 ; 8. ENC - vaesenc XDATA6, [KEYS6 + 16*8] ; 8. ENC - vaesenc XDATA7, [KEYS7 + 16*8] ; 8. ENC - - vaesenc XDATA0, [KEYS0 + 16*9] ; 9. ENC - vaesenc XDATA1, [KEYS1 + 16*9] ; 9. ENC - vaesenc XDATA2, [KEYS2 + 16*9] ; 9. ENC - vaesenc XDATA3, [KEYS3 + 16*9] ; 9. ENC - vaesenc XDATA4, [KEYS4 + 16*9] ; 9. ENC - vaesenc XDATA5, [KEYS5 + 16*9] ; 9. ENC -%ifnidn %%MODE, CBC_XCBC_MAC - mov TMP, [ARG + %%ARG_OUT + 8*0] -%endif - vaesenc XDATA6, XKEY6_9 ; 9. ENC - vaesenc XDATA7, [KEYS7 + 16*9] ; 9. ENC - - vaesenclast XDATA0, [KEYS0 + 16*10] ; 10. ENC - vaesenclast XDATA1, [KEYS1 + 16*10] ; 10. ENC - vaesenclast XDATA2, [KEYS2 + 16*10] ; 10. ENC - vaesenclast XDATA3, [KEYS3 + 16*10] ; 10. ENC - vaesenclast XDATA4, [KEYS4 + 16*10] ; 10. ENC - vaesenclast XDATA5, [KEYS5 + 16*10] ; 10. ENC - vaesenclast XDATA6, [KEYS6 + 16*10] ; 10. ENC - vaesenclast XDATA7, [KEYS7 + 16*10] ; 10. ENC - -%ifnidn %%MODE, CBC_XCBC_MAC - VMOVDQ [TMP], XDATA0 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*1] - VMOVDQ [TMP], XDATA1 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*2] - VMOVDQ [TMP], XDATA2 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*3] - VMOVDQ [TMP], XDATA3 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*4] - VMOVDQ [TMP], XDATA4 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*5] - VMOVDQ [TMP], XDATA5 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*6] - VMOVDQ [TMP], XDATA6 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*7] - VMOVDQ [TMP], XDATA7 ; write back ciphertext -%endif - cmp [LEN_AREA], IDX - jle %%_done - -%%_main_loop: - mov TMP, [ARG + %%ARG_IN + 8*1] - VPXOR2 XDATA0, [IN0 + IDX] ; load next block of plain text - VPXOR2 XDATA1, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + %%ARG_IN + 8*3] - VPXOR2 XDATA2, [IN2 + IDX] ; load next block of plain text - VPXOR2 XDATA3, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + %%ARG_IN + 8*5] - VPXOR2 XDATA4, [IN4 + IDX] ; load next block of plain text - VPXOR2 XDATA5, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + %%ARG_IN + 8*7] - VPXOR2 XDATA6, [IN6 + IDX] ; load next block of plain text - VPXOR2 XDATA7, [TMP + IDX] ; load next block of plain text - - VPXOR2 XDATA0, [KEYS0 + 16*0] ; 0. ARK - VPXOR2 XDATA1, [KEYS1 + 16*0] ; 0. ARK - VPXOR2 XDATA2, [KEYS2 + 16*0] ; 0. ARK - VPXOR2 XDATA3, [KEYS3 + 16*0] ; 0. ARK - VPXOR2 XDATA4, [KEYS4 + 16*0] ; 0. ARK - VPXOR2 XDATA5, [KEYS5 + 16*0] ; 0. ARK - VPXOR2 XDATA6, [KEYS6 + 16*0] ; 0. ARK - VPXOR2 XDATA7, [KEYS7 + 16*0] ; 0. ARK - - vaesenc XDATA0, [KEYS0 + 16*1] ; 1. ENC - vaesenc XDATA1, [KEYS1 + 16*1] ; 1. ENC - vaesenc XDATA2, [KEYS2 + 16*1] ; 1. ENC - vaesenc XDATA3, [KEYS3 + 16*1] ; 1. ENC - vaesenc XDATA4, [KEYS4 + 16*1] ; 1. ENC - vaesenc XDATA5, [KEYS5 + 16*1] ; 1. ENC - vaesenc XDATA6, [KEYS6 + 16*1] ; 1. ENC - vaesenc XDATA7, [KEYS7 + 16*1] ; 1. ENC - - vaesenc XDATA0, [KEYS0 + 16*2] ; 2. ENC - vaesenc XDATA1, [KEYS1 + 16*2] ; 2. ENC - vaesenc XDATA2, [KEYS2 + 16*2] ; 2. ENC - vaesenc XDATA3, [KEYS3 + 16*2] ; 2. ENC - vaesenc XDATA4, [KEYS4 + 16*2] ; 2. ENC - vaesenc XDATA5, [KEYS5 + 16*2] ; 2. ENC - vaesenc XDATA6, [KEYS6 + 16*2] ; 2. ENC - vaesenc XDATA7, [KEYS7 + 16*2] ; 2. ENC - - vaesenc XDATA0, XKEY0_3 ; 3. ENC - vaesenc XDATA1, [KEYS1 + 16*3] ; 3. ENC - vaesenc XDATA2, [KEYS2 + 16*3] ; 3. ENC - vaesenc XDATA3, [KEYS3 + 16*3] ; 3. ENC - vaesenc XDATA4, [KEYS4 + 16*3] ; 3. ENC - vaesenc XDATA5, [KEYS5 + 16*3] ; 3. ENC - vaesenc XDATA6, [KEYS6 + 16*3] ; 3. ENC - vaesenc XDATA7, [KEYS7 + 16*3] ; 3. ENC - - vaesenc XDATA0, [KEYS0 + 16*4] ; 4. ENC - vaesenc XDATA1, XKEY1_4 ; 4. ENC - vaesenc XDATA2, [KEYS2 + 16*4] ; 4. ENC - vaesenc XDATA3, [KEYS3 + 16*4] ; 4. ENC - vaesenc XDATA4, [KEYS4 + 16*4] ; 4. ENC - vaesenc XDATA5, [KEYS5 + 16*4] ; 4. ENC - vaesenc XDATA6, [KEYS6 + 16*4] ; 4. ENC - vaesenc XDATA7, [KEYS7 + 16*4] ; 4. ENC - - vaesenc XDATA0, [KEYS0 + 16*5] ; 5. ENC - vaesenc XDATA1, [KEYS1 + 16*5] ; 5. ENC - vaesenc XDATA2, XKEY2_5 ; 5. ENC - vaesenc XDATA3, [KEYS3 + 16*5] ; 5. ENC - vaesenc XDATA4, [KEYS4 + 16*5] ; 5. ENC - vaesenc XDATA5, [KEYS5 + 16*5] ; 5. ENC - vaesenc XDATA6, [KEYS6 + 16*5] ; 5. ENC - vaesenc XDATA7, [KEYS7 + 16*5] ; 5. ENC - - vaesenc XDATA0, [KEYS0 + 16*6] ; 6. ENC - vaesenc XDATA1, [KEYS1 + 16*6] ; 6. ENC - vaesenc XDATA2, [KEYS2 + 16*6] ; 6. ENC - vaesenc XDATA3, XKEY3_6 ; 6. ENC - vaesenc XDATA4, [KEYS4 + 16*6] ; 6. ENC - vaesenc XDATA5, [KEYS5 + 16*6] ; 6. ENC - vaesenc XDATA6, [KEYS6 + 16*6] ; 6. ENC - vaesenc XDATA7, [KEYS7 + 16*6] ; 6. ENC - - vaesenc XDATA0, [KEYS0 + 16*7] ; 7. ENC - vaesenc XDATA1, [KEYS1 + 16*7] ; 7. ENC - vaesenc XDATA2, [KEYS2 + 16*7] ; 7. ENC - vaesenc XDATA3, [KEYS3 + 16*7] ; 7. ENC - vaesenc XDATA4, XKEY4_7 ; 7. ENC - vaesenc XDATA5, [KEYS5 + 16*7] ; 7. ENC - vaesenc XDATA6, [KEYS6 + 16*7] ; 7. ENC - vaesenc XDATA7, [KEYS7 + 16*7] ; 7. ENC - - vaesenc XDATA0, [KEYS0 + 16*8] ; 8. ENC - vaesenc XDATA1, [KEYS1 + 16*8] ; 8. ENC - vaesenc XDATA2, [KEYS2 + 16*8] ; 8. ENC - vaesenc XDATA3, [KEYS3 + 16*8] ; 8. ENC - vaesenc XDATA4, [KEYS4 + 16*8] ; 8. ENC - vaesenc XDATA5, XKEY5_8 ; 8. ENC - vaesenc XDATA6, [KEYS6 + 16*8] ; 8. ENC - vaesenc XDATA7, [KEYS7 + 16*8] ; 8. ENC - - vaesenc XDATA0, [KEYS0 + 16*9] ; 9. ENC - vaesenc XDATA1, [KEYS1 + 16*9] ; 9. ENC - vaesenc XDATA2, [KEYS2 + 16*9] ; 9. ENC - vaesenc XDATA3, [KEYS3 + 16*9] ; 9. ENC - vaesenc XDATA4, [KEYS4 + 16*9] ; 9. ENC - vaesenc XDATA5, [KEYS5 + 16*9] ; 9. ENC -%ifnidn %%MODE, CBC_XCBC_MAC - mov TMP, [ARG + %%ARG_OUT + 8*0] -%endif - vaesenc XDATA6, XKEY6_9 ; 9. ENC - vaesenc XDATA7, [KEYS7 + 16*9] ; 9. ENC - - vaesenclast XDATA0, [KEYS0 + 16*10] ; 10. ENC - vaesenclast XDATA1, [KEYS1 + 16*10] ; 10. ENC - vaesenclast XDATA2, [KEYS2 + 16*10] ; 10. ENC - vaesenclast XDATA3, [KEYS3 + 16*10] ; 10. ENC - vaesenclast XDATA4, [KEYS4 + 16*10] ; 10. ENC - vaesenclast XDATA5, [KEYS5 + 16*10] ; 10. ENC - vaesenclast XDATA6, [KEYS6 + 16*10] ; 10. ENC - vaesenclast XDATA7, [KEYS7 + 16*10] ; 10. ENC - -%ifnidn %%MODE, CBC_XCBC_MAC - ;; no ciphertext write back for CBC-MAC - VMOVDQ [TMP + IDX], XDATA0 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*1] - VMOVDQ [TMP + IDX], XDATA1 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*2] - VMOVDQ [TMP + IDX], XDATA2 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*3] - VMOVDQ [TMP + IDX], XDATA3 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*4] - VMOVDQ [TMP + IDX], XDATA4 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*5] - VMOVDQ [TMP + IDX], XDATA5 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*6] - VMOVDQ [TMP + IDX], XDATA6 ; write back ciphertext - mov TMP, [ARG + %%ARG_OUT + 8*7] - VMOVDQ [TMP + IDX], XDATA7 ; write back ciphertext -%endif - add IDX, %%OFFSET - cmp [LEN_AREA], IDX - ja %%_main_loop - -%%_done: - ;; update IV for AES128-CBC / store digest for CBC-MAC - vmovdqa [ARG + %%ARG_IV + 16*0], XDATA0 - vmovdqa [ARG + %%ARG_IV + 16*1], XDATA1 - vmovdqa [ARG + %%ARG_IV + 16*2], XDATA2 - vmovdqa [ARG + %%ARG_IV + 16*3], XDATA3 - vmovdqa [ARG + %%ARG_IV + 16*4], XDATA4 - vmovdqa [ARG + %%ARG_IV + 16*5], XDATA5 - vmovdqa [ARG + %%ARG_IV + 16*6], XDATA6 - vmovdqa [ARG + %%ARG_IV + 16*7], XDATA7 - - ;; update IN and OUT - vmovd xmm0, [LEN_AREA] - vpshufd xmm0, xmm0, 0x44 - vpaddq xmm1, xmm0, [ARG + %%ARG_IN + 16*0] - vpaddq xmm2, xmm0, [ARG + %%ARG_IN + 16*1] - vpaddq xmm3, xmm0, [ARG + %%ARG_IN + 16*2] - vpaddq xmm4, xmm0, [ARG + %%ARG_IN + 16*3] - vmovdqa [ARG + %%ARG_IN + 16*0], xmm1 - vmovdqa [ARG + %%ARG_IN + 16*1], xmm2 - vmovdqa [ARG + %%ARG_IN + 16*2], xmm3 - vmovdqa [ARG + %%ARG_IN + 16*3], xmm4 -%ifnidn %%MODE, CBC_XCBC_MAC - vpaddq xmm5, xmm0, [ARG + %%ARG_OUT + 16*0] - vpaddq xmm6, xmm0, [ARG + %%ARG_OUT + 16*1] - vpaddq xmm7, xmm0, [ARG + %%ARG_OUT + 16*2] - vpaddq xmm8, xmm0, [ARG + %%ARG_OUT + 16*3] - vmovdqa [ARG + %%ARG_OUT + 16*0], xmm5 - vmovdqa [ARG + %%ARG_OUT + 16*1], xmm6 - vmovdqa [ARG + %%ARG_OUT + 16*2], xmm7 - vmovdqa [ARG + %%ARG_OUT + 16*3], xmm8 -%endif - - ;; XMMs are saved at a higher level - mov rbp, [GPR_SAVE_AREA + 8*0] -%ifidn %%MODE, CBC_XCBC_MAC - mov rbx, [GPR_SAVE_AREA + 8*1] - mov r12, [GPR_SAVE_AREA + 8*2] - mov r13, [GPR_SAVE_AREA + 8*3] - mov r14, [GPR_SAVE_AREA + 8*4] - mov r15, [GPR_SAVE_AREA + 8*5] -%ifndef LINUX - mov rsi, [GPR_SAVE_AREA + 8*6] - mov rdi, [GPR_SAVE_AREA + 8*7] -%endif -%endif - - add rsp, STACK_size - -%ifdef SAFE_DATA - clear_all_xmms_avx_asm -%endif ;; SAFE_DATA - -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; AES-CBC 128 encrypt macro defines -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%ifndef FUNC -%define FUNC aes_cbc_enc_128_x8 -%define MODE CBC -%define OFFSET 16 -%define ARG_IN _aesarg_in -%define ARG_OUT _aesarg_out -%define ARG_KEYS _aesarg_keys -%define ARG_IV _aesarg_IV -%endif ;; FUNC - -MKGLOBAL(FUNC,function,internal) -FUNC: -%ifdef ARG_OUT - AES_CBC_X8 MODE, OFFSET, ARG_IV, ARG_KEYS, ARG_IN, ARG_OUT -%else - AES_CBC_X8 MODE, OFFSET, ARG_IV, ARG_KEYS, ARG_IN -%endif +align 64 +MKGLOBAL(aes_cbc_enc_128_x8,function,internal) +aes_cbc_enc_128_x8: + AES_CBC_X8 CBC, 9, 16, {arg1 + _aesarg_IV}, {arg1 + _aesarg_keys}, {arg1 + _aesarg_in}, {arg1 + _aesarg_out} ret mksection stack-noexec diff --git a/lib/avx_t1/aes128_cbc_mac_x8_avx.asm b/lib/avx_t1/aes128_cbc_mac_x8_avx.asm index 870da05ee711f1c20361302df1b4d763628c40f3..622ad0210d735f8ef957a91a62caac6005c8dd24 100644 --- a/lib/avx_t1/aes128_cbc_mac_x8_avx.asm +++ b/lib/avx_t1/aes128_cbc_mac_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,16 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -;;; Routine to compute CBC-MAC. It is based on 128 bit CBC AES encrypt code. +;; AES-CMAC-128 -%define FUNC aes128_cbc_mac_x8 -%define MODE CBC_XCBC_MAC -%define OFFSET 16 -%define ARG_IN _aesarg_in -%define ARG_KEYS _aesarg_keys -%define ARG_IV _aesarg_IV +%include "include/aes_cbc_enc_x8_avx.inc" -%include "avx_t1/aes128_cbc_enc_x8_avx.asm" +mksection .text + +align 64 +MKGLOBAL(aes128_cbc_mac_x8,function,internal) +aes128_cbc_mac_x8: + AES_CBC_X8 CBC_XCBC_MAC, 9, 16, {arg1 + _aesarg_IV}, {arg1 + _aesarg_keys}, {arg1 + _aesarg_in} + ret + +mksection stack-noexec diff --git a/lib/avx_t1/aes128_cbcs_1_9_dec_by8_avx.asm b/lib/avx_t1/aes128_cbcs_1_9_dec_by8_avx.asm index b0982c2f61048a64f3139072cb2938ce477e66f7..be8a60adbb8a823e8ac2ad6e49a7991f7fa32224 100644 --- a/lib/avx_t1/aes128_cbcs_1_9_dec_by8_avx.asm +++ b/lib/avx_t1/aes128_cbcs_1_9_dec_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/aes128_cbcs_1_9_enc_x8_avx.asm b/lib/avx_t1/aes128_cbcs_1_9_enc_x8_avx.asm index d3051f2fbad7e97832998e88047e234e2f8ffe0c..e364dde1d1db047e87bcf09852f32c359bcae28b 100644 --- a/lib/avx_t1/aes128_cbcs_1_9_enc_x8_avx.asm +++ b/lib/avx_t1/aes128_cbcs_1_9_enc_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -30,23 +30,16 @@ ;;; skip the following 9 blocks processing 4 buffers at a time. ;;; Updates In and Out pointers at the end. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; struct AES_ARGS { -;; void* in[8]; -;; void* out[8]; -;; UINT128* keys[8]; -;; UINT128 IV[8]; -;; } -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; void aes_cbcs_1_9_enc_128_x8(AES_ARGS *args, UINT64 len); -;; arg 1: ARG : addr of AES_ARGS structure -;; arg 2: LEN : len (in units of bytes) -%define FUNC aes_cbcs_1_9_enc_128_x8 -%define MODE CBC -%define OFFSET 160 -%define ARG_IN _aesarg_in -%define ARG_OUT _aesarg_out -%define ARG_KEYS _aesarg_keys -%define ARG_IV _aesarg_IV +;; AES-CBCS-128 1:9 -%include "avx_t1/aes128_cbc_enc_x8_avx.asm" +%include "include/aes_cbc_enc_x8_avx.inc" + +mksection .text + +align 64 +MKGLOBAL(aes_cbcs_1_9_enc_128_x8,function,internal) +aes_cbcs_1_9_enc_128_x8: + AES_CBC_X8 CBC, 9, 160, {arg1 + _aesarg_IV}, {arg1 + _aesarg_keys}, {arg1 + _aesarg_in}, {arg1 + _aesarg_out} + ret + +mksection stack-noexec diff --git a/lib/avx_t1/aes128_cntr_by8_avx.asm b/lib/avx_t1/aes128_cntr_by8_avx.asm index a74b8599399dd5cce6aceb4b8de54b73716659ed..af1407bf3220f9ba9990326e85d8dd0983fed07c 100644 --- a/lib/avx_t1/aes128_cntr_by8_avx.asm +++ b/lib/avx_t1/aes128_cntr_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ; routine to do AES128 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level diff --git a/lib/avx_t1/aes128_cntr_ccm_by8_avx.asm b/lib/avx_t1/aes128_cntr_ccm_by8_avx.asm index 37bb556c354d2802ffb40351e41d1d78debe6c0e..92229bb0a7a593b8f847484fcb204fc0e921a94e 100644 --- a/lib/avx_t1/aes128_cntr_ccm_by8_avx.asm +++ b/lib/avx_t1/aes128_cntr_ccm_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/aes128_ecb_by8_avx.asm b/lib/avx_t1/aes128_ecb_by8_avx.asm index 65cd39726fe454d004d0d13d2584798b71e9bf34..de2ba008b17ad2f05b913ae5b16b40145325cc2a 100644 --- a/lib/avx_t1/aes128_ecb_by8_avx.asm +++ b/lib/avx_t1/aes128_ecb_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,9 +27,9 @@ ; routine to do AES ECB 128 encrypt/decrypt on 16n bytes doing AES by 8 -%include "include/os.asm" -%include "include/clear_regs.asm" -%include "include/aes_common.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" %ifdef LINUX %define IN rdi diff --git a/lib/avx_t1/aes128_xcbc_mac_x8_avx.asm b/lib/avx_t1/aes128_xcbc_mac_x8_avx.asm index 9d9ec41417b8d3c5aed489c38b3086d13ded07e5..64e669c4dab08c737d88fd895d3528e1e9aa4309 100644 --- a/lib/avx_t1/aes128_xcbc_mac_x8_avx.asm +++ b/lib/avx_t1/aes128_xcbc_mac_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,16 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -;;; routine to do 128 bit AES XCBC +;; AES-XCBC-128 -%define FUNC aes_xcbc_mac_128_x8 -%define MODE CBC_XCBC_MAC -%define OFFSET 16 -%define ARG_IN _aesxcbcarg_in -%define ARG_KEYS _aesxcbcarg_keys -%define ARG_IV _aesxcbcarg_ICV +%include "include/aes_cbc_enc_x8_avx.inc" -%include "avx_t1/aes128_cbc_enc_x8_avx.asm" +mksection .text + +align 64 +MKGLOBAL(aes_xcbc_mac_128_x8,function,internal) +aes_xcbc_mac_128_x8: + AES_CBC_X8 CBC_XCBC_MAC, 9, 16, {arg1 + _aesxcbcarg_ICV}, {arg1 + _aesxcbcarg_keys}, {arg1 + _aesxcbcarg_in} + ret + +mksection stack-noexec diff --git a/lib/avx_t1/aes192_cbc_dec_by8_avx.asm b/lib/avx_t1/aes192_cbc_dec_by8_avx.asm index 13f85efb6521996a1c2c5d613980f1e9466f1d1e..276eb9fdc41317a1136d5c9af7d2f04c14ef324a 100644 --- a/lib/avx_t1/aes192_cbc_dec_by8_avx.asm +++ b/lib/avx_t1/aes192_cbc_dec_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -28,8 +28,8 @@ ; routine to do AES192 CBC decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu diff --git a/lib/avx_t1/aes192_cbc_enc_x8_avx.asm b/lib/avx_t1/aes192_cbc_enc_x8_avx.asm index ac3822d35d08064ddb3e74e7a632d98318e757e0..f4bf275b46c62e5a4c1c702410d35bb80b619942 100644 --- a/lib/avx_t1/aes192_cbc_enc_x8_avx.asm +++ b/lib/avx_t1/aes192_cbc_enc_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,474 +25,16 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -;;; routine to do a 192 bit CBC AES encrypt +;; AES-CBC-192 -;; clobbers all registers except for ARG1 and rbp - -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" - -%define VMOVDQ vmovdqu ;; assume buffers not aligned - -%macro VPXOR2 2 - vpxor %1, %1, %2 -%endm - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; struct AES_ARGS { -;; void* in[8]; -;; void* out[8]; -;; UINT128* keys[8]; -;; UINT128 IV[8]; -;; } -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; void aes_cbc_enc_192_x8(AES_ARGS *args, UINT64 len); -;; arg 1: ARG : addr of AES_ARGS structure -;; arg 2: LEN : len (in units of bytes) - -struc STACK -_gpr_save: resq 1 -_len: resq 1 -endstruc - -%define GPR_SAVE_AREA rsp + _gpr_save -%define LEN_AREA rsp + _len - -%ifdef LINUX -%define ARG rdi -%define LEN rsi -%define REG3 rcx -%define REG4 rdx -%else -%define ARG rcx -%define LEN rdx -%define REG3 rsi -%define REG4 rdi -%endif - -%define IDX rax -%define TMP rbx - -%define KEYS0 REG3 -%define KEYS1 REG4 -%define KEYS2 rbp -%define KEYS3 r8 -%define KEYS4 r9 -%define KEYS5 r10 -%define KEYS6 r11 -%define KEYS7 r12 - -%define IN0 r13 -%define IN2 r14 -%define IN4 r15 -%define IN6 LEN - -%define XDATA0 xmm0 -%define XDATA1 xmm1 -%define XDATA2 xmm2 -%define XDATA3 xmm3 -%define XDATA4 xmm4 -%define XDATA5 xmm5 -%define XDATA6 xmm6 -%define XDATA7 xmm7 - -%define XKEY0_3 xmm8 -%define XKEY1_4 xmm9 -%define XKEY2_5 xmm10 -%define XKEY3_6 xmm11 -%define XKEY4_7 xmm12 -%define XKEY5_8 xmm13 -%define XKEY6_9 xmm14 -%define XTMP xmm15 +%include "include/aes_cbc_enc_x8_avx.inc" mksection .text +align 64 MKGLOBAL(aes_cbc_enc_192_x8,function,internal) aes_cbc_enc_192_x8: - - sub rsp, STACK_size - mov [GPR_SAVE_AREA + 8*0], rbp - - mov IDX, 16 - mov [LEN_AREA], LEN - - mov IN0, [ARG + _aesarg_in + 8*0] - mov IN2, [ARG + _aesarg_in + 8*2] - mov IN4, [ARG + _aesarg_in + 8*4] - mov IN6, [ARG + _aesarg_in + 8*6] - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - mov TMP, [ARG + _aesarg_in + 8*1] - VMOVDQ XDATA0, [IN0] ; load first block of plain text - VMOVDQ XDATA1, [TMP] ; load first block of plain text - mov TMP, [ARG + _aesarg_in + 8*3] - VMOVDQ XDATA2, [IN2] ; load first block of plain text - VMOVDQ XDATA3, [TMP] ; load first block of plain text - mov TMP, [ARG + _aesarg_in + 8*5] - VMOVDQ XDATA4, [IN4] ; load first block of plain text - VMOVDQ XDATA5, [TMP] ; load first block of plain text - mov TMP, [ARG + _aesarg_in + 8*7] - VMOVDQ XDATA6, [IN6] ; load first block of plain text - VMOVDQ XDATA7, [TMP] ; load first block of plain text - - VPXOR2 XDATA0, [ARG + _aesarg_IV + 16*0] ; plaintext XOR IV - VPXOR2 XDATA1, [ARG + _aesarg_IV + 16*1] ; plaintext XOR IV - VPXOR2 XDATA2, [ARG + _aesarg_IV + 16*2] ; plaintext XOR IV - VPXOR2 XDATA3, [ARG + _aesarg_IV + 16*3] ; plaintext XOR IV - VPXOR2 XDATA4, [ARG + _aesarg_IV + 16*4] ; plaintext XOR IV - VPXOR2 XDATA5, [ARG + _aesarg_IV + 16*5] ; plaintext XOR IV - VPXOR2 XDATA6, [ARG + _aesarg_IV + 16*6] ; plaintext XOR IV - VPXOR2 XDATA7, [ARG + _aesarg_IV + 16*7] ; plaintext XOR IV - - mov KEYS0, [ARG + _aesarg_keys + 8*0] - mov KEYS1, [ARG + _aesarg_keys + 8*1] - mov KEYS2, [ARG + _aesarg_keys + 8*2] - mov KEYS3, [ARG + _aesarg_keys + 8*3] - mov KEYS4, [ARG + _aesarg_keys + 8*4] - mov KEYS5, [ARG + _aesarg_keys + 8*5] - mov KEYS6, [ARG + _aesarg_keys + 8*6] - mov KEYS7, [ARG + _aesarg_keys + 8*7] - - VPXOR2 XDATA0, [KEYS0 + 16*0] ; 0. ARK - VPXOR2 XDATA1, [KEYS1 + 16*0] ; 0. ARK - VPXOR2 XDATA2, [KEYS2 + 16*0] ; 0. ARK - VPXOR2 XDATA3, [KEYS3 + 16*0] ; 0. ARK - VPXOR2 XDATA4, [KEYS4 + 16*0] ; 0. ARK - VPXOR2 XDATA5, [KEYS5 + 16*0] ; 0. ARK - VPXOR2 XDATA6, [KEYS6 + 16*0] ; 0. ARK - VPXOR2 XDATA7, [KEYS7 + 16*0] ; 0. ARK - - vaesenc XDATA0, [KEYS0 + 16*1] ; 1. ENC - vaesenc XDATA1, [KEYS1 + 16*1] ; 1. ENC - vaesenc XDATA2, [KEYS2 + 16*1] ; 1. ENC - vaesenc XDATA3, [KEYS3 + 16*1] ; 1. ENC - vaesenc XDATA4, [KEYS4 + 16*1] ; 1. ENC - vaesenc XDATA5, [KEYS5 + 16*1] ; 1. ENC - vaesenc XDATA6, [KEYS6 + 16*1] ; 1. ENC - vaesenc XDATA7, [KEYS7 + 16*1] ; 1. ENC - - vmovdqa XKEY0_3, [KEYS0 + 16*3] ; load round 3 key - - vaesenc XDATA0, [KEYS0 + 16*2] ; 2. ENC - vaesenc XDATA1, [KEYS1 + 16*2] ; 2. ENC - vaesenc XDATA2, [KEYS2 + 16*2] ; 2. ENC - vaesenc XDATA3, [KEYS3 + 16*2] ; 2. ENC - vaesenc XDATA4, [KEYS4 + 16*2] ; 2. ENC - vaesenc XDATA5, [KEYS5 + 16*2] ; 2. ENC - vaesenc XDATA6, [KEYS6 + 16*2] ; 2. ENC - vaesenc XDATA7, [KEYS7 + 16*2] ; 2. ENC - - vmovdqa XKEY1_4, [KEYS1 + 16*4] ; load round 4 key - - vaesenc XDATA0, XKEY0_3 ; 3. ENC - vaesenc XDATA1, [KEYS1 + 16*3] ; 3. ENC - vaesenc XDATA2, [KEYS2 + 16*3] ; 3. ENC - vaesenc XDATA3, [KEYS3 + 16*3] ; 3. ENC - vaesenc XDATA4, [KEYS4 + 16*3] ; 3. ENC - vaesenc XDATA5, [KEYS5 + 16*3] ; 3. ENC - vaesenc XDATA6, [KEYS6 + 16*3] ; 3. ENC - vaesenc XDATA7, [KEYS7 + 16*3] ; 3. ENC - - vaesenc XDATA0, [KEYS0 + 16*4] ; 4. ENC - vmovdqa XKEY2_5, [KEYS2 + 16*5] ; load round 5 key - vaesenc XDATA1, XKEY1_4 ; 4. ENC - vaesenc XDATA2, [KEYS2 + 16*4] ; 4. ENC - vaesenc XDATA3, [KEYS3 + 16*4] ; 4. ENC - vaesenc XDATA4, [KEYS4 + 16*4] ; 4. ENC - vaesenc XDATA5, [KEYS5 + 16*4] ; 4. ENC - vaesenc XDATA6, [KEYS6 + 16*4] ; 4. ENC - vaesenc XDATA7, [KEYS7 + 16*4] ; 4. ENC - - vaesenc XDATA0, [KEYS0 + 16*5] ; 5. ENC - vaesenc XDATA1, [KEYS1 + 16*5] ; 5. ENC - vmovdqa XKEY3_6, [KEYS3 + 16*6] ; load round 6 key - vaesenc XDATA2, XKEY2_5 ; 5. ENC - vaesenc XDATA3, [KEYS3 + 16*5] ; 5. ENC - vaesenc XDATA4, [KEYS4 + 16*5] ; 5. ENC - vaesenc XDATA5, [KEYS5 + 16*5] ; 5. ENC - vaesenc XDATA6, [KEYS6 + 16*5] ; 5. ENC - vaesenc XDATA7, [KEYS7 + 16*5] ; 5. ENC - - vaesenc XDATA0, [KEYS0 + 16*6] ; 6. ENC - vaesenc XDATA1, [KEYS1 + 16*6] ; 6. ENC - vaesenc XDATA2, [KEYS2 + 16*6] ; 6. ENC - vmovdqa XKEY4_7, [KEYS4 + 16*7] ; load round 7 key - vaesenc XDATA3, XKEY3_6 ; 6. ENC - vaesenc XDATA4, [KEYS4 + 16*6] ; 6. ENC - vaesenc XDATA5, [KEYS5 + 16*6] ; 6. ENC - vaesenc XDATA6, [KEYS6 + 16*6] ; 6. ENC - vaesenc XDATA7, [KEYS7 + 16*6] ; 6. ENC - - vaesenc XDATA0, [KEYS0 + 16*7] ; 7. ENC - vaesenc XDATA1, [KEYS1 + 16*7] ; 7. ENC - vaesenc XDATA2, [KEYS2 + 16*7] ; 7. ENC - vaesenc XDATA3, [KEYS3 + 16*7] ; 7. ENC - vmovdqa XKEY5_8, [KEYS5 + 16*8] ; load round 8 key - vaesenc XDATA4, XKEY4_7 ; 7. ENC - vaesenc XDATA5, [KEYS5 + 16*7] ; 7. ENC - vaesenc XDATA6, [KEYS6 + 16*7] ; 7. ENC - vaesenc XDATA7, [KEYS7 + 16*7] ; 7. ENC - - vaesenc XDATA0, [KEYS0 + 16*8] ; 8. ENC - vaesenc XDATA1, [KEYS1 + 16*8] ; 8. ENC - vaesenc XDATA2, [KEYS2 + 16*8] ; 8. ENC - vaesenc XDATA3, [KEYS3 + 16*8] ; 8. ENC - vaesenc XDATA4, [KEYS4 + 16*8] ; 8. ENC - vmovdqa XKEY6_9, [KEYS6 + 16*9] ; load round 9 key - vaesenc XDATA5, XKEY5_8 ; 8. ENC - vaesenc XDATA6, [KEYS6 + 16*8] ; 8. ENC - vaesenc XDATA7, [KEYS7 + 16*8] ; 8. ENC - - vaesenc XDATA0, [KEYS0 + 16*9] ; 9. ENC - vaesenc XDATA1, [KEYS1 + 16*9] ; 9. ENC - vaesenc XDATA2, [KEYS2 + 16*9] ; 9. ENC - vaesenc XDATA3, [KEYS3 + 16*9] ; 9. ENC - vaesenc XDATA4, [KEYS4 + 16*9] ; 9. ENC - vaesenc XDATA5, [KEYS5 + 16*9] ; 9. ENC - mov TMP, [ARG + _aesarg_out + 8*0] - vaesenc XDATA6, XKEY6_9 ; 9. ENC - vaesenc XDATA7, [KEYS7 + 16*9] ; 9. ENC - - vaesenc XDATA0, [KEYS0 + 16*10] ; 10. ENC - vaesenc XDATA1, [KEYS1 + 16*10] ; 10. ENC - vaesenc XDATA2, [KEYS2 + 16*10] ; 10. ENC - vaesenc XDATA3, [KEYS3 + 16*10] ; 10. ENC - vaesenc XDATA4, [KEYS4 + 16*10] ; 10. ENC - vaesenc XDATA5, [KEYS5 + 16*10] ; 10. ENC - vaesenc XDATA6, [KEYS6 + 16*10] ; 10. ENC - vaesenc XDATA7, [KEYS7 + 16*10] ; 10. ENC - - vaesenc XDATA0, [KEYS0 + 16*11] ; 11. ENC - vaesenc XDATA1, [KEYS1 + 16*11] ; 11. ENC - vaesenc XDATA2, [KEYS2 + 16*11] ; 11. ENC - vaesenc XDATA3, [KEYS3 + 16*11] ; 11. ENC - vaesenc XDATA4, [KEYS4 + 16*11] ; 11. ENC - vaesenc XDATA5, [KEYS5 + 16*11] ; 11. ENC - vaesenc XDATA6, [KEYS6 + 16*11] ; 11. ENC - vaesenc XDATA7, [KEYS7 + 16*11] ; 11. ENC - - vaesenclast XDATA0, [KEYS0 + 16*12] ; 12. ENC - vaesenclast XDATA1, [KEYS1 + 16*12] ; 12. ENC - vaesenclast XDATA2, [KEYS2 + 16*12] ; 12. ENC - vaesenclast XDATA3, [KEYS3 + 16*12] ; 12. ENC - vaesenclast XDATA4, [KEYS4 + 16*12] ; 12. ENC - vaesenclast XDATA5, [KEYS5 + 16*12] ; 12. ENC - vaesenclast XDATA6, [KEYS6 + 16*12] ; 12. ENC - vaesenclast XDATA7, [KEYS7 + 16*12] ; 12. ENC - - VMOVDQ [TMP], XDATA0 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*1] - VMOVDQ [TMP], XDATA1 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*2] - VMOVDQ [TMP], XDATA2 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*3] - VMOVDQ [TMP], XDATA3 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*4] - VMOVDQ [TMP], XDATA4 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*5] - VMOVDQ [TMP], XDATA5 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*6] - VMOVDQ [TMP], XDATA6 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*7] - VMOVDQ [TMP], XDATA7 ; write back ciphertext - - cmp [LEN_AREA], IDX - je done - -main_loop: - mov TMP, [ARG + _aesarg_in + 8*1] - VPXOR2 XDATA0, [IN0 + IDX] ; load next block of plain text - VPXOR2 XDATA1, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + _aesarg_in + 8*3] - VPXOR2 XDATA2, [IN2 + IDX] ; load next block of plain text - VPXOR2 XDATA3, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + _aesarg_in + 8*5] - VPXOR2 XDATA4, [IN4 + IDX] ; load next block of plain text - VPXOR2 XDATA5, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + _aesarg_in + 8*7] - VPXOR2 XDATA6, [IN6 + IDX] ; load next block of plain text - VPXOR2 XDATA7, [TMP + IDX] ; load next block of plain text - - VPXOR2 XDATA0, [KEYS0 + 16*0] ; 0. ARK - VPXOR2 XDATA1, [KEYS1 + 16*0] ; 0. ARK - VPXOR2 XDATA2, [KEYS2 + 16*0] ; 0. ARK - VPXOR2 XDATA3, [KEYS3 + 16*0] ; 0. ARK - VPXOR2 XDATA4, [KEYS4 + 16*0] ; 0. ARK - VPXOR2 XDATA5, [KEYS5 + 16*0] ; 0. ARK - VPXOR2 XDATA6, [KEYS6 + 16*0] ; 0. ARK - VPXOR2 XDATA7, [KEYS7 + 16*0] ; 0. ARK - - vaesenc XDATA0, [KEYS0 + 16*1] ; 1. ENC - vaesenc XDATA1, [KEYS1 + 16*1] ; 1. ENC - vaesenc XDATA2, [KEYS2 + 16*1] ; 1. ENC - vaesenc XDATA3, [KEYS3 + 16*1] ; 1. ENC - vaesenc XDATA4, [KEYS4 + 16*1] ; 1. ENC - vaesenc XDATA5, [KEYS5 + 16*1] ; 1. ENC - vaesenc XDATA6, [KEYS6 + 16*1] ; 1. ENC - vaesenc XDATA7, [KEYS7 + 16*1] ; 1. ENC - - vaesenc XDATA0, [KEYS0 + 16*2] ; 2. ENC - vaesenc XDATA1, [KEYS1 + 16*2] ; 2. ENC - vaesenc XDATA2, [KEYS2 + 16*2] ; 2. ENC - vaesenc XDATA3, [KEYS3 + 16*2] ; 2. ENC - vaesenc XDATA4, [KEYS4 + 16*2] ; 2. ENC - vaesenc XDATA5, [KEYS5 + 16*2] ; 2. ENC - vaesenc XDATA6, [KEYS6 + 16*2] ; 2. ENC - vaesenc XDATA7, [KEYS7 + 16*2] ; 2. ENC - - vaesenc XDATA0, XKEY0_3 ; 3. ENC - vaesenc XDATA1, [KEYS1 + 16*3] ; 3. ENC - vaesenc XDATA2, [KEYS2 + 16*3] ; 3. ENC - vaesenc XDATA3, [KEYS3 + 16*3] ; 3. ENC - vaesenc XDATA4, [KEYS4 + 16*3] ; 3. ENC - vaesenc XDATA5, [KEYS5 + 16*3] ; 3. ENC - vaesenc XDATA6, [KEYS6 + 16*3] ; 3. ENC - vaesenc XDATA7, [KEYS7 + 16*3] ; 3. ENC - - vaesenc XDATA0, [KEYS0 + 16*4] ; 4. ENC - vaesenc XDATA1, XKEY1_4 ; 4. ENC - vaesenc XDATA2, [KEYS2 + 16*4] ; 4. ENC - vaesenc XDATA3, [KEYS3 + 16*4] ; 4. ENC - vaesenc XDATA4, [KEYS4 + 16*4] ; 4. ENC - vaesenc XDATA5, [KEYS5 + 16*4] ; 4. ENC - vaesenc XDATA6, [KEYS6 + 16*4] ; 4. ENC - vaesenc XDATA7, [KEYS7 + 16*4] ; 4. ENC - - vaesenc XDATA0, [KEYS0 + 16*5] ; 5. ENC - vaesenc XDATA1, [KEYS1 + 16*5] ; 5. ENC - vaesenc XDATA2, XKEY2_5 ; 5. ENC - vaesenc XDATA3, [KEYS3 + 16*5] ; 5. ENC - vaesenc XDATA4, [KEYS4 + 16*5] ; 5. ENC - vaesenc XDATA5, [KEYS5 + 16*5] ; 5. ENC - vaesenc XDATA6, [KEYS6 + 16*5] ; 5. ENC - vaesenc XDATA7, [KEYS7 + 16*5] ; 5. ENC - - vaesenc XDATA0, [KEYS0 + 16*6] ; 6. ENC - vaesenc XDATA1, [KEYS1 + 16*6] ; 6. ENC - vaesenc XDATA2, [KEYS2 + 16*6] ; 6. ENC - vaesenc XDATA3, XKEY3_6 ; 6. ENC - vaesenc XDATA4, [KEYS4 + 16*6] ; 6. ENC - vaesenc XDATA5, [KEYS5 + 16*6] ; 6. ENC - vaesenc XDATA6, [KEYS6 + 16*6] ; 6. ENC - vaesenc XDATA7, [KEYS7 + 16*6] ; 6. ENC - - vaesenc XDATA0, [KEYS0 + 16*7] ; 7. ENC - vaesenc XDATA1, [KEYS1 + 16*7] ; 7. ENC - vaesenc XDATA2, [KEYS2 + 16*7] ; 7. ENC - vaesenc XDATA3, [KEYS3 + 16*7] ; 7. ENC - vaesenc XDATA4, XKEY4_7 ; 7. ENC - vaesenc XDATA5, [KEYS5 + 16*7] ; 7. ENC - vaesenc XDATA6, [KEYS6 + 16*7] ; 7. ENC - vaesenc XDATA7, [KEYS7 + 16*7] ; 7. ENC - - vaesenc XDATA0, [KEYS0 + 16*8] ; 8. ENC - vaesenc XDATA1, [KEYS1 + 16*8] ; 8. ENC - vaesenc XDATA2, [KEYS2 + 16*8] ; 8. ENC - vaesenc XDATA3, [KEYS3 + 16*8] ; 8. ENC - vaesenc XDATA4, [KEYS4 + 16*8] ; 8. ENC - vaesenc XDATA5, XKEY5_8 ; 8. ENC - vaesenc XDATA6, [KEYS6 + 16*8] ; 8. ENC - vaesenc XDATA7, [KEYS7 + 16*8] ; 8. ENC - - vaesenc XDATA0, [KEYS0 + 16*9] ; 9. ENC - vaesenc XDATA1, [KEYS1 + 16*9] ; 9. ENC - vaesenc XDATA2, [KEYS2 + 16*9] ; 9. ENC - vaesenc XDATA3, [KEYS3 + 16*9] ; 9. ENC - vaesenc XDATA4, [KEYS4 + 16*9] ; 9. ENC - vaesenc XDATA5, [KEYS5 + 16*9] ; 9. ENC - mov TMP, [ARG + _aesarg_out + 8*0] - vaesenc XDATA6, XKEY6_9 ; 9. ENC - vaesenc XDATA7, [KEYS7 + 16*9] ; 9. ENC - - vaesenc XDATA0, [KEYS0 + 16*10] ; 10. ENC - vaesenc XDATA1, [KEYS1 + 16*10] ; 10. ENC - vaesenc XDATA2, [KEYS2 + 16*10] ; 10. ENC - vaesenc XDATA3, [KEYS3 + 16*10] ; 10. ENC - vaesenc XDATA4, [KEYS4 + 16*10] ; 10. ENC - vaesenc XDATA5, [KEYS5 + 16*10] ; 10. ENC - vaesenc XDATA6, [KEYS6 + 16*10] ; 10. ENC - vaesenc XDATA7, [KEYS7 + 16*10] ; 10. ENC - - vaesenc XDATA0, [KEYS0 + 16*11] ; 11. ENC - vaesenc XDATA1, [KEYS1 + 16*11] ; 11. ENC - vaesenc XDATA2, [KEYS2 + 16*11] ; 11. ENC - vaesenc XDATA3, [KEYS3 + 16*11] ; 11. ENC - vaesenc XDATA4, [KEYS4 + 16*11] ; 11. ENC - vaesenc XDATA5, [KEYS5 + 16*11] ; 11. ENC - vaesenc XDATA6, [KEYS6 + 16*11] ; 11. ENC - vaesenc XDATA7, [KEYS7 + 16*11] ; 11. ENC - - vaesenclast XDATA0, [KEYS0 + 16*12] ; 12. ENC - vaesenclast XDATA1, [KEYS1 + 16*12] ; 12. ENC - vaesenclast XDATA2, [KEYS2 + 16*12] ; 12. ENC - vaesenclast XDATA3, [KEYS3 + 16*12] ; 12. ENC - vaesenclast XDATA4, [KEYS4 + 16*12] ; 12. ENC - vaesenclast XDATA5, [KEYS5 + 16*12] ; 12. ENC - vaesenclast XDATA6, [KEYS6 + 16*12] ; 12. ENC - vaesenclast XDATA7, [KEYS7 + 16*12] ; 12. ENC - - VMOVDQ [TMP + IDX], XDATA0 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*1] - VMOVDQ [TMP + IDX], XDATA1 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*2] - VMOVDQ [TMP + IDX], XDATA2 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*3] - VMOVDQ [TMP + IDX], XDATA3 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*4] - VMOVDQ [TMP + IDX], XDATA4 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*5] - VMOVDQ [TMP + IDX], XDATA5 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*6] - VMOVDQ [TMP + IDX], XDATA6 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*7] - VMOVDQ [TMP + IDX], XDATA7 ; write back ciphertext - - add IDX, 16 - cmp [LEN_AREA], IDX - jne main_loop - -done: - ;; update IV - vmovdqa [ARG + _aesarg_IV + 16*0], XDATA0 - vmovdqa [ARG + _aesarg_IV + 16*1], XDATA1 - vmovdqa [ARG + _aesarg_IV + 16*2], XDATA2 - vmovdqa [ARG + _aesarg_IV + 16*3], XDATA3 - vmovdqa [ARG + _aesarg_IV + 16*4], XDATA4 - vmovdqa [ARG + _aesarg_IV + 16*5], XDATA5 - vmovdqa [ARG + _aesarg_IV + 16*6], XDATA6 - vmovdqa [ARG + _aesarg_IV + 16*7], XDATA7 - - ;; update IN and OUT - vmovd xmm0, [LEN_AREA] - vpshufd xmm0, xmm0, 0x44 - vpaddq xmm1, xmm0, [ARG + _aesarg_in + 16*0] - vpaddq xmm2, xmm0, [ARG + _aesarg_in + 16*1] - vpaddq xmm3, xmm0, [ARG + _aesarg_in + 16*2] - vpaddq xmm4, xmm0, [ARG + _aesarg_in + 16*3] - vmovdqa [ARG + _aesarg_in + 16*0], xmm1 - vmovdqa [ARG + _aesarg_in + 16*1], xmm2 - vmovdqa [ARG + _aesarg_in + 16*2], xmm3 - vmovdqa [ARG + _aesarg_in + 16*3], xmm4 - vpaddq xmm5, xmm0, [ARG + _aesarg_out + 16*0] - vpaddq xmm6, xmm0, [ARG + _aesarg_out + 16*1] - vpaddq xmm7, xmm0, [ARG + _aesarg_out + 16*2] - vpaddq xmm8, xmm0, [ARG + _aesarg_out + 16*3] - vmovdqa [ARG + _aesarg_out + 16*0], xmm5 - vmovdqa [ARG + _aesarg_out + 16*1], xmm6 - vmovdqa [ARG + _aesarg_out + 16*2], xmm7 - vmovdqa [ARG + _aesarg_out + 16*3], xmm8 - -;; XMMs are saved at a higher level - mov rbp, [GPR_SAVE_AREA + 8*0] - - add rsp, STACK_size - -%ifdef SAFE_DATA - clear_all_xmms_avx_asm -%endif ;; SAFE_DATA - - ret + AES_CBC_X8 CBC, 11, 16, {arg1 + _aesarg_IV}, {arg1 + _aesarg_keys}, {arg1 + _aesarg_in}, {arg1 + _aesarg_out} + ret mksection stack-noexec diff --git a/lib/avx_t1/aes192_cntr_by8_avx.asm b/lib/avx_t1/aes192_cntr_by8_avx.asm index c6e928c0b4878d3ac6dd2551226849b319a7ae97..577c8c9e5ac8120d421f8e7597932fbdc0c0acbf 100644 --- a/lib/avx_t1/aes192_cntr_by8_avx.asm +++ b/lib/avx_t1/aes192_cntr_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ; routine to do AES192 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level diff --git a/lib/avx_t1/aes192_ecb_by8_avx.asm b/lib/avx_t1/aes192_ecb_by8_avx.asm index b7402bade15c25b98bad8c5875c11b5403e19052..7b7f7fdc6f1d1f7158cd1ba4b4b87148a7c86b02 100644 --- a/lib/avx_t1/aes192_ecb_by8_avx.asm +++ b/lib/avx_t1/aes192_ecb_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,5 +29,5 @@ %define AES_ECB_NROUNDS 12 -%include "include/os.asm" +%include "include/os.inc" %include "avx_t1/aes128_ecb_by8_avx.asm" diff --git a/lib/avx_t1/aes192_gcm_by8_avx.asm b/lib/avx_t1/aes192_gcm_by8_avx.asm deleted file mode 100644 index e072d850bd4f9c46c4750b848b279ce6478ced03..0000000000000000000000000000000000000000 --- a/lib/avx_t1/aes192_gcm_by8_avx.asm +++ /dev/null @@ -1,31 +0,0 @@ -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022 Intel Corporation All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions -; are met: -; * Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; * Redistributions in binary form must reproduce the above copyright -; notice, this list of conditions and the following disclaimer in -; the documentation and/or other materials provided with the -; distribution. -; * Neither the name of Intel Corporation nor the names of its -; contributors may be used to endorse or promote products derived -; from this software without specific prior written permission. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%define GCM192_MODE 1 -%include "avx_t1/gcm_avx_gen2.asm" diff --git a/lib/avx_t1/aes256_cbc_dec_by8_avx.asm b/lib/avx_t1/aes256_cbc_dec_by8_avx.asm index 36aa543875a38369316dadb39bc6994b42398124..78b80f6bbb11ff0bd88e454631dcc3266c78e78c 100644 --- a/lib/avx_t1/aes256_cbc_dec_by8_avx.asm +++ b/lib/avx_t1/aes256_cbc_dec_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -28,8 +28,8 @@ ; routine to do AES256 CBC decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu diff --git a/lib/avx_t1/aes256_cbc_enc_x8_avx.asm b/lib/avx_t1/aes256_cbc_enc_x8_avx.asm index a4eb28fb12c1d5b0c8420744cfebfa519acc30b5..c61ed1810fa70d9e7f35e2b2a1d73e768b40304f 100644 --- a/lib/avx_t1/aes256_cbc_enc_x8_avx.asm +++ b/lib/avx_t1/aes256_cbc_enc_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,539 +25,16 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -;;; routine to do a 256 bit CBC AES encrypt +;; AES-CBC-256 -;; clobbers all registers except for ARG1 and rbp - -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" - -%define VMOVDQ vmovdqu ;; assume buffers not aligned - -%macro VPXOR2 2 - vpxor %1, %1, %2 -%endm - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; struct AES_ARGS { -;; void* in[8]; -;; void* out[8]; -;; UINT128* keys[8]; -;; UINT128 IV[8]; -;; } -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; void aes_cbc_enc_256_x8(AES_ARGS *args, UINT64 len); -;; arg 1: ARG : addr of AES_ARGS structure -;; arg 2: LEN : len (in units of bytes) - -struc STACK -_gpr_save: resq 8 -_len: resq 1 -endstruc - -%define GPR_SAVE_AREA rsp + _gpr_save -%define LEN_AREA rsp + _len - -%ifdef LINUX -%define ARG rdi -%define LEN rsi -%define REG3 rcx -%define REG4 rdx -%else -%define ARG rcx -%define LEN rdx -%define REG3 rsi -%define REG4 rdi -%endif - -%define IDX rax -%define TMP rbx - -%define KEYS0 REG3 -%define KEYS1 REG4 -%define KEYS2 rbp -%define KEYS3 r8 -%define KEYS4 r9 -%define KEYS5 r10 -%define KEYS6 r11 -%define KEYS7 r12 - -%define IN0 r13 -%define IN2 r14 -%define IN4 r15 -%define IN6 LEN - -%define XDATA0 xmm0 -%define XDATA1 xmm1 -%define XDATA2 xmm2 -%define XDATA3 xmm3 -%define XDATA4 xmm4 -%define XDATA5 xmm5 -%define XDATA6 xmm6 -%define XDATA7 xmm7 - -%define XKEY0_3 xmm8 -%define XKEY1_4 xmm9 -%define XKEY2_5 xmm10 -%define XKEY3_6 xmm11 -%define XKEY4_7 xmm12 -%define XKEY5_8 xmm13 -%define XKEY6_9 xmm14 -%define XTMP xmm15 +%include "include/aes_cbc_enc_x8_avx.inc" mksection .text -%ifdef CBC_MAC -MKGLOBAL(aes256_cbc_mac_x8,function,internal) -aes256_cbc_mac_x8: -%else + +align 64 MKGLOBAL(aes_cbc_enc_256_x8,function,internal) aes_cbc_enc_256_x8: -%endif - sub rsp, STACK_size - mov [GPR_SAVE_AREA + 8*0], rbp -%ifdef CBC_MAC - mov [GPR_SAVE_AREA + 8*1], rbx - mov [GPR_SAVE_AREA + 8*2], r12 - mov [GPR_SAVE_AREA + 8*3], r13 - mov [GPR_SAVE_AREA + 8*4], r14 - mov [GPR_SAVE_AREA + 8*5], r15 -%ifndef LINUX - mov [GPR_SAVE_AREA + 8*6], rsi - mov [GPR_SAVE_AREA + 8*7], rdi -%endif -%endif - - mov IDX, 16 - mov [LEN_AREA], LEN - - mov IN0, [ARG + _aesarg_in + 8*0] - mov IN2, [ARG + _aesarg_in + 8*2] - mov IN4, [ARG + _aesarg_in + 8*4] - mov IN6, [ARG + _aesarg_in + 8*6] - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - mov TMP, [ARG + _aesarg_in + 8*1] - VMOVDQ XDATA0, [IN0] ; load first block of plain text - VMOVDQ XDATA1, [TMP] ; load first block of plain text - mov TMP, [ARG + _aesarg_in + 8*3] - VMOVDQ XDATA2, [IN2] ; load first block of plain text - VMOVDQ XDATA3, [TMP] ; load first block of plain text - mov TMP, [ARG + _aesarg_in + 8*5] - VMOVDQ XDATA4, [IN4] ; load first block of plain text - VMOVDQ XDATA5, [TMP] ; load first block of plain text - mov TMP, [ARG + _aesarg_in + 8*7] - VMOVDQ XDATA6, [IN6] ; load first block of plain text - VMOVDQ XDATA7, [TMP] ; load first block of plain text - - VPXOR2 XDATA0, [ARG + _aesarg_IV + 16*0] ; plaintext XOR IV - VPXOR2 XDATA1, [ARG + _aesarg_IV + 16*1] ; plaintext XOR IV - VPXOR2 XDATA2, [ARG + _aesarg_IV + 16*2] ; plaintext XOR IV - VPXOR2 XDATA3, [ARG + _aesarg_IV + 16*3] ; plaintext XOR IV - VPXOR2 XDATA4, [ARG + _aesarg_IV + 16*4] ; plaintext XOR IV - VPXOR2 XDATA5, [ARG + _aesarg_IV + 16*5] ; plaintext XOR IV - VPXOR2 XDATA6, [ARG + _aesarg_IV + 16*6] ; plaintext XOR IV - VPXOR2 XDATA7, [ARG + _aesarg_IV + 16*7] ; plaintext XOR IV - - mov KEYS0, [ARG + _aesarg_keys + 8*0] - mov KEYS1, [ARG + _aesarg_keys + 8*1] - mov KEYS2, [ARG + _aesarg_keys + 8*2] - mov KEYS3, [ARG + _aesarg_keys + 8*3] - mov KEYS4, [ARG + _aesarg_keys + 8*4] - mov KEYS5, [ARG + _aesarg_keys + 8*5] - mov KEYS6, [ARG + _aesarg_keys + 8*6] - mov KEYS7, [ARG + _aesarg_keys + 8*7] - - VPXOR2 XDATA0, [KEYS0 + 16*0] ; 0. ARK - VPXOR2 XDATA1, [KEYS1 + 16*0] ; 0. ARK - VPXOR2 XDATA2, [KEYS2 + 16*0] ; 0. ARK - VPXOR2 XDATA3, [KEYS3 + 16*0] ; 0. ARK - VPXOR2 XDATA4, [KEYS4 + 16*0] ; 0. ARK - VPXOR2 XDATA5, [KEYS5 + 16*0] ; 0. ARK - VPXOR2 XDATA6, [KEYS6 + 16*0] ; 0. ARK - VPXOR2 XDATA7, [KEYS7 + 16*0] ; 0. ARK - - vaesenc XDATA0, [KEYS0 + 16*1] ; 1. ENC - vaesenc XDATA1, [KEYS1 + 16*1] ; 1. ENC - vaesenc XDATA2, [KEYS2 + 16*1] ; 1. ENC - vaesenc XDATA3, [KEYS3 + 16*1] ; 1. ENC - vaesenc XDATA4, [KEYS4 + 16*1] ; 1. ENC - vaesenc XDATA5, [KEYS5 + 16*1] ; 1. ENC - vaesenc XDATA6, [KEYS6 + 16*1] ; 1. ENC - vaesenc XDATA7, [KEYS7 + 16*1] ; 1. ENC - - vmovdqa XKEY0_3, [KEYS0 + 16*3] ; load round 3 key - - vaesenc XDATA0, [KEYS0 + 16*2] ; 2. ENC - vaesenc XDATA1, [KEYS1 + 16*2] ; 2. ENC - vaesenc XDATA2, [KEYS2 + 16*2] ; 2. ENC - vaesenc XDATA3, [KEYS3 + 16*2] ; 2. ENC - vaesenc XDATA4, [KEYS4 + 16*2] ; 2. ENC - vaesenc XDATA5, [KEYS5 + 16*2] ; 2. ENC - vaesenc XDATA6, [KEYS6 + 16*2] ; 2. ENC - vaesenc XDATA7, [KEYS7 + 16*2] ; 2. ENC - - vmovdqa XKEY1_4, [KEYS1 + 16*4] ; load round 4 key - - vaesenc XDATA0, XKEY0_3 ; 3. ENC - vaesenc XDATA1, [KEYS1 + 16*3] ; 3. ENC - vaesenc XDATA2, [KEYS2 + 16*3] ; 3. ENC - vaesenc XDATA3, [KEYS3 + 16*3] ; 3. ENC - vaesenc XDATA4, [KEYS4 + 16*3] ; 3. ENC - vaesenc XDATA5, [KEYS5 + 16*3] ; 3. ENC - vaesenc XDATA6, [KEYS6 + 16*3] ; 3. ENC - vaesenc XDATA7, [KEYS7 + 16*3] ; 3. ENC - - vaesenc XDATA0, [KEYS0 + 16*4] ; 4. ENC - vmovdqa XKEY2_5, [KEYS2 + 16*5] ; load round 5 key - vaesenc XDATA1, XKEY1_4 ; 4. ENC - vaesenc XDATA2, [KEYS2 + 16*4] ; 4. ENC - vaesenc XDATA3, [KEYS3 + 16*4] ; 4. ENC - vaesenc XDATA4, [KEYS4 + 16*4] ; 4. ENC - vaesenc XDATA5, [KEYS5 + 16*4] ; 4. ENC - vaesenc XDATA6, [KEYS6 + 16*4] ; 4. ENC - vaesenc XDATA7, [KEYS7 + 16*4] ; 4. ENC - - vaesenc XDATA0, [KEYS0 + 16*5] ; 5. ENC - vaesenc XDATA1, [KEYS1 + 16*5] ; 5. ENC - vmovdqa XKEY3_6, [KEYS3 + 16*6] ; load round 6 key - vaesenc XDATA2, XKEY2_5 ; 5. ENC - vaesenc XDATA3, [KEYS3 + 16*5] ; 5. ENC - vaesenc XDATA4, [KEYS4 + 16*5] ; 5. ENC - vaesenc XDATA5, [KEYS5 + 16*5] ; 5. ENC - vaesenc XDATA6, [KEYS6 + 16*5] ; 5. ENC - vaesenc XDATA7, [KEYS7 + 16*5] ; 5. ENC - - vaesenc XDATA0, [KEYS0 + 16*6] ; 6. ENC - vaesenc XDATA1, [KEYS1 + 16*6] ; 6. ENC - vaesenc XDATA2, [KEYS2 + 16*6] ; 6. ENC - vmovdqa XKEY4_7, [KEYS4 + 16*7] ; load round 7 key - vaesenc XDATA3, XKEY3_6 ; 6. ENC - vaesenc XDATA4, [KEYS4 + 16*6] ; 6. ENC - vaesenc XDATA5, [KEYS5 + 16*6] ; 6. ENC - vaesenc XDATA6, [KEYS6 + 16*6] ; 6. ENC - vaesenc XDATA7, [KEYS7 + 16*6] ; 6. ENC - - vaesenc XDATA0, [KEYS0 + 16*7] ; 7. ENC - vaesenc XDATA1, [KEYS1 + 16*7] ; 7. ENC - vaesenc XDATA2, [KEYS2 + 16*7] ; 7. ENC - vaesenc XDATA3, [KEYS3 + 16*7] ; 7. ENC - vmovdqa XKEY5_8, [KEYS5 + 16*8] ; load round 8 key - vaesenc XDATA4, XKEY4_7 ; 7. ENC - vaesenc XDATA5, [KEYS5 + 16*7] ; 7. ENC - vaesenc XDATA6, [KEYS6 + 16*7] ; 7. ENC - vaesenc XDATA7, [KEYS7 + 16*7] ; 7. ENC - - vaesenc XDATA0, [KEYS0 + 16*8] ; 8. ENC - vaesenc XDATA1, [KEYS1 + 16*8] ; 8. ENC - vaesenc XDATA2, [KEYS2 + 16*8] ; 8. ENC - vaesenc XDATA3, [KEYS3 + 16*8] ; 8. ENC - vaesenc XDATA4, [KEYS4 + 16*8] ; 8. ENC - vmovdqa XKEY6_9, [KEYS6 + 16*9] ; load round 9 key - vaesenc XDATA5, XKEY5_8 ; 8. ENC - vaesenc XDATA6, [KEYS6 + 16*8] ; 8. ENC - vaesenc XDATA7, [KEYS7 + 16*8] ; 8. ENC - - vaesenc XDATA0, [KEYS0 + 16*9] ; 9. ENC - vaesenc XDATA1, [KEYS1 + 16*9] ; 9. ENC - vaesenc XDATA2, [KEYS2 + 16*9] ; 9. ENC - vaesenc XDATA3, [KEYS3 + 16*9] ; 9. ENC - vaesenc XDATA4, [KEYS4 + 16*9] ; 9. ENC - vaesenc XDATA5, [KEYS5 + 16*9] ; 9. ENC - mov TMP, [ARG + _aesarg_out + 8*0] - vaesenc XDATA6, XKEY6_9 ; 9. ENC - vaesenc XDATA7, [KEYS7 + 16*9] ; 9. ENC - - vaesenc XDATA0, [KEYS0 + 16*10] ; 10. ENC - vaesenc XDATA1, [KEYS1 + 16*10] ; 10. ENC - vaesenc XDATA2, [KEYS2 + 16*10] ; 10. ENC - vaesenc XDATA3, [KEYS3 + 16*10] ; 10. ENC - vaesenc XDATA4, [KEYS4 + 16*10] ; 10. ENC - vaesenc XDATA5, [KEYS5 + 16*10] ; 10. ENC - vaesenc XDATA6, [KEYS6 + 16*10] ; 10. ENC - vaesenc XDATA7, [KEYS7 + 16*10] ; 10. ENC - - vaesenc XDATA0, [KEYS0 + 16*11] ; 11. ENC - vaesenc XDATA1, [KEYS1 + 16*11] ; 11. ENC - vaesenc XDATA2, [KEYS2 + 16*11] ; 11. ENC - vaesenc XDATA3, [KEYS3 + 16*11] ; 11. ENC - vaesenc XDATA4, [KEYS4 + 16*11] ; 11. ENC - vaesenc XDATA5, [KEYS5 + 16*11] ; 11. ENC - vaesenc XDATA6, [KEYS6 + 16*11] ; 11. ENC - vaesenc XDATA7, [KEYS7 + 16*11] ; 11. ENC - - vaesenc XDATA0, [KEYS0 + 16*12] ; 12. ENC - vaesenc XDATA1, [KEYS1 + 16*12] ; 12. ENC - vaesenc XDATA2, [KEYS2 + 16*12] ; 12. ENC - vaesenc XDATA3, [KEYS3 + 16*12] ; 12. ENC - vaesenc XDATA4, [KEYS4 + 16*12] ; 12. ENC - vaesenc XDATA5, [KEYS5 + 16*12] ; 12. ENC - vaesenc XDATA6, [KEYS6 + 16*12] ; 12. ENC - vaesenc XDATA7, [KEYS7 + 16*12] ; 12. ENC - - vaesenc XDATA0, [KEYS0 + 16*13] ; 13. ENC - vaesenc XDATA1, [KEYS1 + 16*13] ; 13. ENC - vaesenc XDATA2, [KEYS2 + 16*13] ; 13. ENC - vaesenc XDATA3, [KEYS3 + 16*13] ; 13. ENC - vaesenc XDATA4, [KEYS4 + 16*13] ; 13. ENC - vaesenc XDATA5, [KEYS5 + 16*13] ; 13. ENC - vaesenc XDATA6, [KEYS6 + 16*13] ; 13. ENC - vaesenc XDATA7, [KEYS7 + 16*13] ; 13. ENC - - vaesenclast XDATA0, [KEYS0 + 16*14] ; 14. ENC - vaesenclast XDATA1, [KEYS1 + 16*14] ; 14. ENC - vaesenclast XDATA2, [KEYS2 + 16*14] ; 14. ENC - vaesenclast XDATA3, [KEYS3 + 16*14] ; 14. ENC - vaesenclast XDATA4, [KEYS4 + 16*14] ; 14. ENC - vaesenclast XDATA5, [KEYS5 + 16*14] ; 14. ENC - vaesenclast XDATA6, [KEYS6 + 16*14] ; 14. ENC - vaesenclast XDATA7, [KEYS7 + 16*14] ; 14. ENC - -%ifndef CBC_MAC - VMOVDQ [TMP], XDATA0 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*1] - VMOVDQ [TMP], XDATA1 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*2] - VMOVDQ [TMP], XDATA2 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*3] - VMOVDQ [TMP], XDATA3 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*4] - VMOVDQ [TMP], XDATA4 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*5] - VMOVDQ [TMP], XDATA5 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*6] - VMOVDQ [TMP], XDATA6 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*7] - VMOVDQ [TMP], XDATA7 ; write back ciphertext -%endif - cmp [LEN_AREA], IDX - je done - -main_loop: - mov TMP, [ARG + _aesarg_in + 8*1] - VPXOR2 XDATA0, [IN0 + IDX] ; load next block of plain text - VPXOR2 XDATA1, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + _aesarg_in + 8*3] - VPXOR2 XDATA2, [IN2 + IDX] ; load next block of plain text - VPXOR2 XDATA3, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + _aesarg_in + 8*5] - VPXOR2 XDATA4, [IN4 + IDX] ; load next block of plain text - VPXOR2 XDATA5, [TMP + IDX] ; load next block of plain text - mov TMP, [ARG + _aesarg_in + 8*7] - VPXOR2 XDATA6, [IN6 + IDX] ; load next block of plain text - VPXOR2 XDATA7, [TMP + IDX] ; load next block of plain text - - VPXOR2 XDATA0, [KEYS0 + 16*0] ; 0. ARK - VPXOR2 XDATA1, [KEYS1 + 16*0] ; 0. ARK - VPXOR2 XDATA2, [KEYS2 + 16*0] ; 0. ARK - VPXOR2 XDATA3, [KEYS3 + 16*0] ; 0. ARK - VPXOR2 XDATA4, [KEYS4 + 16*0] ; 0. ARK - VPXOR2 XDATA5, [KEYS5 + 16*0] ; 0. ARK - VPXOR2 XDATA6, [KEYS6 + 16*0] ; 0. ARK - VPXOR2 XDATA7, [KEYS7 + 16*0] ; 0. ARK - - vaesenc XDATA0, [KEYS0 + 16*1] ; 1. ENC - vaesenc XDATA1, [KEYS1 + 16*1] ; 1. ENC - vaesenc XDATA2, [KEYS2 + 16*1] ; 1. ENC - vaesenc XDATA3, [KEYS3 + 16*1] ; 1. ENC - vaesenc XDATA4, [KEYS4 + 16*1] ; 1. ENC - vaesenc XDATA5, [KEYS5 + 16*1] ; 1. ENC - vaesenc XDATA6, [KEYS6 + 16*1] ; 1. ENC - vaesenc XDATA7, [KEYS7 + 16*1] ; 1. ENC - - vaesenc XDATA0, [KEYS0 + 16*2] ; 2. ENC - vaesenc XDATA1, [KEYS1 + 16*2] ; 2. ENC - vaesenc XDATA2, [KEYS2 + 16*2] ; 2. ENC - vaesenc XDATA3, [KEYS3 + 16*2] ; 2. ENC - vaesenc XDATA4, [KEYS4 + 16*2] ; 2. ENC - vaesenc XDATA5, [KEYS5 + 16*2] ; 2. ENC - vaesenc XDATA6, [KEYS6 + 16*2] ; 2. ENC - vaesenc XDATA7, [KEYS7 + 16*2] ; 2. ENC - - vaesenc XDATA0, XKEY0_3 ; 3. ENC - vaesenc XDATA1, [KEYS1 + 16*3] ; 3. ENC - vaesenc XDATA2, [KEYS2 + 16*3] ; 3. ENC - vaesenc XDATA3, [KEYS3 + 16*3] ; 3. ENC - vaesenc XDATA4, [KEYS4 + 16*3] ; 3. ENC - vaesenc XDATA5, [KEYS5 + 16*3] ; 3. ENC - vaesenc XDATA6, [KEYS6 + 16*3] ; 3. ENC - vaesenc XDATA7, [KEYS7 + 16*3] ; 3. ENC - - vaesenc XDATA0, [KEYS0 + 16*4] ; 4. ENC - vaesenc XDATA1, XKEY1_4 ; 4. ENC - vaesenc XDATA2, [KEYS2 + 16*4] ; 4. ENC - vaesenc XDATA3, [KEYS3 + 16*4] ; 4. ENC - vaesenc XDATA4, [KEYS4 + 16*4] ; 4. ENC - vaesenc XDATA5, [KEYS5 + 16*4] ; 4. ENC - vaesenc XDATA6, [KEYS6 + 16*4] ; 4. ENC - vaesenc XDATA7, [KEYS7 + 16*4] ; 4. ENC - - vaesenc XDATA0, [KEYS0 + 16*5] ; 5. ENC - vaesenc XDATA1, [KEYS1 + 16*5] ; 5. ENC - vaesenc XDATA2, XKEY2_5 ; 5. ENC - vaesenc XDATA3, [KEYS3 + 16*5] ; 5. ENC - vaesenc XDATA4, [KEYS4 + 16*5] ; 5. ENC - vaesenc XDATA5, [KEYS5 + 16*5] ; 5. ENC - vaesenc XDATA6, [KEYS6 + 16*5] ; 5. ENC - vaesenc XDATA7, [KEYS7 + 16*5] ; 5. ENC - - vaesenc XDATA0, [KEYS0 + 16*6] ; 6. ENC - vaesenc XDATA1, [KEYS1 + 16*6] ; 6. ENC - vaesenc XDATA2, [KEYS2 + 16*6] ; 6. ENC - vaesenc XDATA3, XKEY3_6 ; 6. ENC - vaesenc XDATA4, [KEYS4 + 16*6] ; 6. ENC - vaesenc XDATA5, [KEYS5 + 16*6] ; 6. ENC - vaesenc XDATA6, [KEYS6 + 16*6] ; 6. ENC - vaesenc XDATA7, [KEYS7 + 16*6] ; 6. ENC - - vaesenc XDATA0, [KEYS0 + 16*7] ; 7. ENC - vaesenc XDATA1, [KEYS1 + 16*7] ; 7. ENC - vaesenc XDATA2, [KEYS2 + 16*7] ; 7. ENC - vaesenc XDATA3, [KEYS3 + 16*7] ; 7. ENC - vaesenc XDATA4, XKEY4_7 ; 7. ENC - vaesenc XDATA5, [KEYS5 + 16*7] ; 7. ENC - vaesenc XDATA6, [KEYS6 + 16*7] ; 7. ENC - vaesenc XDATA7, [KEYS7 + 16*7] ; 7. ENC - - vaesenc XDATA0, [KEYS0 + 16*8] ; 8. ENC - vaesenc XDATA1, [KEYS1 + 16*8] ; 8. ENC - vaesenc XDATA2, [KEYS2 + 16*8] ; 8. ENC - vaesenc XDATA3, [KEYS3 + 16*8] ; 8. ENC - vaesenc XDATA4, [KEYS4 + 16*8] ; 8. ENC - vaesenc XDATA5, XKEY5_8 ; 8. ENC - vaesenc XDATA6, [KEYS6 + 16*8] ; 8. ENC - vaesenc XDATA7, [KEYS7 + 16*8] ; 8. ENC - - vaesenc XDATA0, [KEYS0 + 16*9] ; 9. ENC - vaesenc XDATA1, [KEYS1 + 16*9] ; 9. ENC - vaesenc XDATA2, [KEYS2 + 16*9] ; 9. ENC - vaesenc XDATA3, [KEYS3 + 16*9] ; 9. ENC - vaesenc XDATA4, [KEYS4 + 16*9] ; 9. ENC - vaesenc XDATA5, [KEYS5 + 16*9] ; 9. ENC - mov TMP, [ARG + _aesarg_out + 8*0] - vaesenc XDATA6, XKEY6_9 ; 9. ENC - vaesenc XDATA7, [KEYS7 + 16*9] ; 9. ENC - - vaesenc XDATA0, [KEYS0 + 16*10] ; 10. ENC - vaesenc XDATA1, [KEYS1 + 16*10] ; 10. ENC - vaesenc XDATA2, [KEYS2 + 16*10] ; 10. ENC - vaesenc XDATA3, [KEYS3 + 16*10] ; 10. ENC - vaesenc XDATA4, [KEYS4 + 16*10] ; 10. ENC - vaesenc XDATA5, [KEYS5 + 16*10] ; 10. ENC - vaesenc XDATA6, [KEYS6 + 16*10] ; 10. ENC - vaesenc XDATA7, [KEYS7 + 16*10] ; 10. ENC - - vaesenc XDATA0, [KEYS0 + 16*11] ; 11. ENC - vaesenc XDATA1, [KEYS1 + 16*11] ; 11. ENC - vaesenc XDATA2, [KEYS2 + 16*11] ; 11. ENC - vaesenc XDATA3, [KEYS3 + 16*11] ; 11. ENC - vaesenc XDATA4, [KEYS4 + 16*11] ; 11. ENC - vaesenc XDATA5, [KEYS5 + 16*11] ; 11. ENC - vaesenc XDATA6, [KEYS6 + 16*11] ; 11. ENC - vaesenc XDATA7, [KEYS7 + 16*11] ; 11. ENC - - vaesenc XDATA0, [KEYS0 + 16*12] ; 12. ENC - vaesenc XDATA1, [KEYS1 + 16*12] ; 12. ENC - vaesenc XDATA2, [KEYS2 + 16*12] ; 12. ENC - vaesenc XDATA3, [KEYS3 + 16*12] ; 12. ENC - vaesenc XDATA4, [KEYS4 + 16*12] ; 12. ENC - vaesenc XDATA5, [KEYS5 + 16*12] ; 12. ENC - vaesenc XDATA6, [KEYS6 + 16*12] ; 12. ENC - vaesenc XDATA7, [KEYS7 + 16*12] ; 12. ENC - - vaesenc XDATA0, [KEYS0 + 16*13] ; 13. ENC - vaesenc XDATA1, [KEYS1 + 16*13] ; 13. ENC - vaesenc XDATA2, [KEYS2 + 16*13] ; 13. ENC - vaesenc XDATA3, [KEYS3 + 16*13] ; 13. ENC - vaesenc XDATA4, [KEYS4 + 16*13] ; 13. ENC - vaesenc XDATA5, [KEYS5 + 16*13] ; 13. ENC - vaesenc XDATA6, [KEYS6 + 16*13] ; 13. ENC - vaesenc XDATA7, [KEYS7 + 16*13] ; 13. ENC - - vaesenclast XDATA0, [KEYS0 + 16*14] ; 14. ENC - vaesenclast XDATA1, [KEYS1 + 16*14] ; 14. ENC - vaesenclast XDATA2, [KEYS2 + 16*14] ; 14. ENC - vaesenclast XDATA3, [KEYS3 + 16*14] ; 14. ENC - vaesenclast XDATA4, [KEYS4 + 16*14] ; 14. ENC - vaesenclast XDATA5, [KEYS5 + 16*14] ; 14. ENC - vaesenclast XDATA6, [KEYS6 + 16*14] ; 14. ENC - vaesenclast XDATA7, [KEYS7 + 16*14] ; 14. ENC - -%ifndef CBC_MAC - VMOVDQ [TMP + IDX], XDATA0 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*1] - VMOVDQ [TMP + IDX], XDATA1 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*2] - VMOVDQ [TMP + IDX], XDATA2 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*3] - VMOVDQ [TMP + IDX], XDATA3 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*4] - VMOVDQ [TMP + IDX], XDATA4 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*5] - VMOVDQ [TMP + IDX], XDATA5 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*6] - VMOVDQ [TMP + IDX], XDATA6 ; write back ciphertext - mov TMP, [ARG + _aesarg_out + 8*7] - VMOVDQ [TMP + IDX], XDATA7 ; write back ciphertext -%endif - add IDX, 16 - cmp [LEN_AREA], IDX - jne main_loop - -done: - ;; update IV - vmovdqa [ARG + _aesarg_IV + 16*0], XDATA0 - vmovdqa [ARG + _aesarg_IV + 16*1], XDATA1 - vmovdqa [ARG + _aesarg_IV + 16*2], XDATA2 - vmovdqa [ARG + _aesarg_IV + 16*3], XDATA3 - vmovdqa [ARG + _aesarg_IV + 16*4], XDATA4 - vmovdqa [ARG + _aesarg_IV + 16*5], XDATA5 - vmovdqa [ARG + _aesarg_IV + 16*6], XDATA6 - vmovdqa [ARG + _aesarg_IV + 16*7], XDATA7 - - ;; update IN and OUT - vmovd xmm0, [LEN_AREA] - vpshufd xmm0, xmm0, 0x44 - vpaddq xmm1, xmm0, [ARG + _aesarg_in + 16*0] - vpaddq xmm2, xmm0, [ARG + _aesarg_in + 16*1] - vpaddq xmm3, xmm0, [ARG + _aesarg_in + 16*2] - vpaddq xmm4, xmm0, [ARG + _aesarg_in + 16*3] - vmovdqa [ARG + _aesarg_in + 16*0], xmm1 - vmovdqa [ARG + _aesarg_in + 16*1], xmm2 - vmovdqa [ARG + _aesarg_in + 16*2], xmm3 - vmovdqa [ARG + _aesarg_in + 16*3], xmm4 -%ifndef CBC_MAC - vpaddq xmm5, xmm0, [ARG + _aesarg_out + 16*0] - vpaddq xmm6, xmm0, [ARG + _aesarg_out + 16*1] - vpaddq xmm7, xmm0, [ARG + _aesarg_out + 16*2] - vpaddq xmm8, xmm0, [ARG + _aesarg_out + 16*3] - vmovdqa [ARG + _aesarg_out + 16*0], xmm5 - vmovdqa [ARG + _aesarg_out + 16*1], xmm6 - vmovdqa [ARG + _aesarg_out + 16*2], xmm7 - vmovdqa [ARG + _aesarg_out + 16*3], xmm8 -%endif - -;; XMMs are saved at a higher level - mov rbp, [GPR_SAVE_AREA + 8*0] -%ifdef CBC_MAC - mov rbx, [GPR_SAVE_AREA + 8*1] - mov r12, [GPR_SAVE_AREA + 8*2] - mov r13, [GPR_SAVE_AREA + 8*3] - mov r14, [GPR_SAVE_AREA + 8*4] - mov r15, [GPR_SAVE_AREA + 8*5] -%ifndef LINUX - mov rsi, [GPR_SAVE_AREA + 8*6] - mov rdi, [GPR_SAVE_AREA + 8*7] -%endif -%endif - - add rsp, STACK_size - -%ifdef SAFE_DATA - clear_all_xmms_avx_asm -%endif ;; SAFE_DATA - - ret + AES_CBC_X8 CBC, 13, 16, {arg1 + _aesarg_IV}, {arg1 + _aesarg_keys}, {arg1 + _aesarg_in}, {arg1 + _aesarg_out} + ret mksection stack-noexec diff --git a/lib/avx_t1/aes256_cbc_mac_x8_avx.asm b/lib/avx_t1/aes256_cbc_mac_x8_avx.asm index 842eea9dfea0974e56dfad29b19c5149bd596b9a..c4e72a2c88f67436af7c5d5f48bcaad8e6996d53 100644 --- a/lib/avx_t1/aes256_cbc_mac_x8_avx.asm +++ b/lib/avx_t1/aes256_cbc_mac_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,7 +25,16 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -;;; Routine to compute CBC-MAC. It is based on 256 bit CBC AES encrypt code. +;; AES-CMAC-256 -%define CBC_MAC 1 -%include "avx_t1/aes256_cbc_enc_x8_avx.asm" +%include "include/aes_cbc_enc_x8_avx.inc" + +mksection .text + +align 64 +MKGLOBAL(aes256_cbc_mac_x8,function,internal) +aes256_cbc_mac_x8: + AES_CBC_X8 CBC_XCBC_MAC, 13, 16, {arg1 + _aesarg_IV}, {arg1 + _aesarg_keys}, {arg1 + _aesarg_in} + ret + +mksection stack-noexec diff --git a/lib/avx_t1/aes256_cntr_by8_avx.asm b/lib/avx_t1/aes256_cntr_by8_avx.asm index 3d5d3b6ae3cd1fa4ea654ef5689ca2bd2559bc07..61502bdffed2fcc72dfc9109be8f7741d22c63f3 100644 --- a/lib/avx_t1/aes256_cntr_by8_avx.asm +++ b/lib/avx_t1/aes256_cntr_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ; routine to do AES256 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level diff --git a/lib/avx_t1/aes256_cntr_ccm_by8_avx.asm b/lib/avx_t1/aes256_cntr_ccm_by8_avx.asm index d51bab557d090c03848a158c87c2fb6411e86875..d038b09d1e9446430fdf09d6b43a7c1d52e0516e 100644 --- a/lib/avx_t1/aes256_cntr_ccm_by8_avx.asm +++ b/lib/avx_t1/aes256_cntr_ccm_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/aes256_ecb_by8_avx.asm b/lib/avx_t1/aes256_ecb_by8_avx.asm index 514ffa89725e6ee4f4e1f58565b176aae2dec5dd..50e87c9131b20af5ec05f2bad74606fad0bf049e 100644 --- a/lib/avx_t1/aes256_ecb_by8_avx.asm +++ b/lib/avx_t1/aes256_ecb_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,5 +29,5 @@ %define AES_ECB_NROUNDS 14 -%include "include/os.asm" +%include "include/os.inc" %include "avx_t1/aes128_ecb_by8_avx.asm" diff --git a/lib/avx_t1/aes256_gcm_by8_avx.asm b/lib/avx_t1/aes256_gcm_by8_avx.asm deleted file mode 100644 index 9d5146e99af6cebc17c27ed9073e7f51d4076f85..0000000000000000000000000000000000000000 --- a/lib/avx_t1/aes256_gcm_by8_avx.asm +++ /dev/null @@ -1,30 +0,0 @@ -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2011-2022 Intel Corporation All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions -; are met: -; * Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; * Redistributions in binary form must reproduce the above copyright -; notice, this list of conditions and the following disclaimer in -; the documentation and/or other materials provided with the -; distribution. -; * Neither the name of Intel Corporation nor the names of its -; contributors may be used to endorse or promote products derived -; from this software without specific prior written permission. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%define GCM256_MODE 1 -%include "avx_t1/gcm_avx_gen2.asm" diff --git a/lib/avx_t1/aes_cfb_avx.asm b/lib/avx_t1/aes_cfb_avx.asm index b1c018889cd42ff76f4a730f9b33862ba1a1312f..3411d573ed7d1722a076a7222bb18b6104325a74 100644 --- a/lib/avx_t1/aes_cfb_avx.asm +++ b/lib/avx_t1/aes_cfb_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" ;;; Routines to do 128/256 bit CFB AES encrypt/decrypt operations on one block only. @@ -207,7 +207,7 @@ align 32 aes_cfb_256_one_avx: aes_cfb_256_one_avx2: aes_cfb_256_one_avx512: - + endbranch64 do_cfb 13 ret diff --git a/lib/avx_t1/aes_ecb_quic_x8_avx.asm b/lib/avx_t1/aes_ecb_quic_x8_avx.asm new file mode 100644 index 0000000000000000000000000000000000000000..b9adb569a594331fd141ee90a43649dd7bab3acf --- /dev/null +++ b/lib/avx_t1/aes_ecb_quic_x8_avx.asm @@ -0,0 +1,239 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB encrypt for QUIC on 16-byte buffers + +; XMM registers are clobbered. Saving/restoring must be done at a higher level + +; void aes_ecb_quic_enc_x_avx(void *in, +; UINT128 keys[], +; void *out, +; UINT64 num_buffers); +; +; x = key size (128/256) +; arg 1: IN: array of pointers to input buffers +; arg 2: KEYS: pointer to keys (common for all buffers) +; arg 3: OUT: array of pointers to output buffers) +; arg 4: N_BUFS: number of 16-byte buffers +; + +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" +%include "include/cet.inc" + +%define AES_ECB_QUIC_ENC_128 aes_ecb_quic_enc_128_avx +%define AES_ECB_QUIC_ENC_256 aes_ecb_quic_enc_256_avx + +;; ============================================================================= +;; Loads 1 AES block from up to 8 buffers into XMM registers +%macro XMM_LOAD_BLOCKS_MULT_IN_0_8 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%ARRAY_INP %2 ; [in] array of input data pointers +%define %%INP %3 ; [clobbered] input data pointer to read 16 bytes from +%define %%DST0 %4 ; [out] XMM register with loaded data +%define %%DST1 %5 ; [out] XMM register with loaded data +%define %%DST2 %6 ; [out] XMM register with loaded data +%define %%DST3 %7 ; [out] XMM register with loaded data +%define %%DST4 %8 ; [out] XMM register with loaded data +%define %%DST5 %9 ; [out] XMM register with loaded data +%define %%DST6 %10 ; [out] XMM register with loaded data +%define %%DST7 %11 ; [out] XMM register with loaded data + +%assign dst_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%DSTREG %%DST %+ dst_idx + mov %%INP, [%%ARRAY_INP + dst_idx*8] + vmovdqu %%DSTREG, [%%INP] +%undef %%DSTREG +%assign dst_idx (dst_idx + 1) +%endrep + +%endmacro + +;; ============================================================================= +;; Stores 1 AES blocks to up to 8 buffers from XMM registers +%macro XMM_STORE_BLOCKS_MULT_OUT_0_8 12 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 8) +%define %%ARRAY_OUTP %2 ; [in] array of output data pointers to write to +%define %%OUTP %3 ; [clobbered] output data pointer to write to +%define %%SRC0 %4 ; [in] XMM register with data to store +%define %%SRC1 %5 ; [in] XMM register with data to store +%define %%SRC2 %6 ; [in] XMM register with data to store +%define %%SRC3 %7 ; [in] XMM register with data to store +%define %%SRC4 %8 ; [in] XMM register with data to store +%define %%SRC5 %9 ; [in] XMM register with data to store +%define %%SRC6 %10 ; [in] XMM register with data to store +%define %%SRC7 %11 ; [in] XMM register with data to store +%define %%TMP %12 ; [clobbered] Temporary GP register + +%assign src_idx 0 +%rep (%%NUM_BLOCKS) +%xdefine %%SRCREG %%SRC %+ src_idx + ; Store 5 bytes to each buffer + mov %%OUTP, [%%ARRAY_OUTP + src_idx*8] + vmovd [%%OUTP], %%SRCREG + vpextrb [%%OUTP + 4], %%SRCREG, 4 +%undef %%SRCREG +%assign src_idx (src_idx + 1) +%endrep + +%endmacro + +%ifdef LINUX +%define IN rdi +%define KEYS rsi +%define OUT rdx +%define N_BUFS rcx +%else +%define IN rcx +%define KEYS rdx +%define OUT r8 +%define N_BUFS r9 +%endif + +%define IDX rax +%define TMP IDX +%define XDATA0 xmm0 +%define XDATA1 xmm1 +%define XDATA2 xmm2 +%define XDATA3 xmm3 +%define XKEY0 xmm4 +%define XKEY2 xmm5 +%define XKEY4 xmm6 +%define XKEY6 xmm7 +%define XKEY10 xmm8 +%define XKEY_A xmm14 +%define XKEY_B xmm15 + +mksection .text + +%macro AES_ECB_QUIC 1 +%define %%NROUNDS %1 ; [in] Number of rounds + +%define IDX rax +%define TMP r11 +%define TMP2 r10 +%define XDATA0 xmm0 +%define XDATA1 xmm1 +%define XDATA2 xmm2 +%define XDATA3 xmm3 +%define XDATA4 xmm4 +%define XDATA5 xmm5 +%define XDATA6 xmm6 +%define XDATA7 xmm7 +%define XKEY1 xmm8 + + or N_BUFS, N_BUFS + jz %%done + xor IDX, IDX + mov TMP, N_BUFS + and TMP, 0x7 ; number of initial buffers (0 to 7 buffers) + jz %%main_loop + ; branch to different code block based on remainder + cmp TMP, 4 + je %%initial_num_buffers_is_4 + jb %%initial_num_buffers_is_3_1 + cmp TMP, 6 + je %%initial_num_buffers_is_6 + jb %%initial_num_buffers_is_5 + ja %%initial_num_buffers_is_7 +%%initial_num_buffers_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2 + ja %%initial_num_buffers_is_3 + je %%initial_num_buffers_is_2 + ;; fall through for `jmp %%initial_num_buffers_is_1` +%assign num_buffers 1 +%rep 7 +%%initial_num_buffers_is_ %+ num_buffers : + ; load initial blocks + XMM_LOAD_BLOCKS_MULT_IN_0_8 num_buffers, IN, TMP, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform aesenc encryption on initial blocks +%rep (%%NROUNDS + 1) ; 10/14 + movdqu XKEY1, [KEYS + %%I*16] + XMM_AESENC_ROUND_BLOCKS_AVX_0_8 XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, num_buffers, (%%NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store initial blocks + XMM_STORE_BLOCKS_MULT_OUT_0_8 num_buffers, OUT, TMP, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7, TMP2 + add IDX, num_buffers + cmp IDX, N_BUFS + je %%done +%assign num_buffers (num_buffers + 1) + jmp %%main_loop +%endrep +align 16 +%%main_loop: + ; load next 8 blocks + XMM_LOAD_BLOCKS_MULT_IN_0_8 8, {IN + IDX*8}, TMP, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform AES encryption/decryption on 8 blocks +%rep (%%NROUNDS + 1) ; 10/14 + movdqu XKEY1, [KEYS + %%I*16] + XMM_AESENC_ROUND_BLOCKS_AVX_0_8 XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, 8, (%%NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store 8 blocks + XMM_STORE_BLOCKS_MULT_OUT_0_8 8, {OUT + IDX*8}, TMP, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7, TMP2 + add IDX, 8 + cmp IDX, N_BUFS + jne %%main_loop +%%done: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif + ret +%endmacro + +align 16 +MKGLOBAL(AES_ECB_QUIC_ENC_128,function,internal) +AES_ECB_QUIC_ENC_128: + endbranch64 + AES_ECB_QUIC 10 + +align 16 +MKGLOBAL(AES_ECB_QUIC_ENC_256,function,internal) +AES_ECB_QUIC_ENC_256: + endbranch64 + AES_ECB_QUIC 14 + +mksection stack-noexec diff --git a/lib/avx_t1/chacha20_avx.asm b/lib/avx_t1/chacha20_avx.asm index ef0bacffe1d7e3f614bf82a40dfeb1cb87a11a5a..d4db354857076fbe8979c4758379c94ca497c73b 100644 --- a/lib/avx_t1/chacha20_avx.asm +++ b/lib/avx_t1/chacha20_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" -%include "include/chacha_poly_defines.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" +%include "include/chacha_poly_defines.inc" mksection .rodata default rel diff --git a/lib/avx_t1/crc16_x25_avx.asm b/lib/avx_t1/crc16_x25_avx.asm index a2f85b58d749e991c64d3af2a52dbd51ea08cd65..8556bdd30492ceaacf2e21e1363b3b583f55749d 100644 --- a/lib/avx_t1/crc16_x25_avx.asm +++ b/lib/avx_t1/crc16_x25_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_refl_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/crc32_by8_avx.asm b/lib/avx_t1/crc32_by8_avx.asm index 2403b0fa4fa69116ddfeb152c10b548fb2b306a4..343747b559c8f0ed4ac7895a12fdbb0ef8211eff 100644 --- a/lib/avx_t1/crc32_by8_avx.asm +++ b/lib/avx_t1/crc32_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -35,11 +35,11 @@ ;; "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" ;; URL: http://download.intel.com/design/intarch/papers/323102.pdf -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/reg_sizes.inc" %include "include/crc32.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" [bits 64] default rel diff --git a/lib/avx_t1/crc32_fp_avx.asm b/lib/avx_t1/crc32_fp_avx.asm index a1fe8de03af9bed4b1bad267b7977f0d73f5f8e1..ba0fd0e60ff45178b35d7da47095c7d9ceecb697 100644 --- a/lib/avx_t1/crc32_fp_avx.asm +++ b/lib/avx_t1/crc32_fp_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/crc32_iuup_avx.asm b/lib/avx_t1/crc32_iuup_avx.asm index 8c283bb1d2dad66eb4cd1e529759d36fc4825e6f..287075a255e6932cc9f3678b74d5615befb9e32b 100644 --- a/lib/avx_t1/crc32_iuup_avx.asm +++ b/lib/avx_t1/crc32_iuup_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/crc32_lte_avx.asm b/lib/avx_t1/crc32_lte_avx.asm index e363bb81987e6ec3854543c70493b7dafa312466..4f0ad631d4fed455142a77b3870371cd17e648b8 100644 --- a/lib/avx_t1/crc32_lte_avx.asm +++ b/lib/avx_t1/crc32_lte_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/crc32_refl_by8_avx.asm b/lib/avx_t1/crc32_refl_by8_avx.asm index bbbd4a0ae5c5c8fa8c03935e831aa0154cc9f606..f7cfa860ef4c93c9361d7113f8709d55e6797701 100644 --- a/lib/avx_t1/crc32_refl_by8_avx.asm +++ b/lib/avx_t1/crc32_refl_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -35,11 +35,11 @@ ;; "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" ;; URL: http://download.intel.com/design/intarch/papers/323102.pdf -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/reg_sizes.inc" %include "include/crc32_refl.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" [bits 64] default rel diff --git a/lib/avx_t1/crc32_sctp_avx.asm b/lib/avx_t1/crc32_sctp_avx.asm index d1a25c615daa283318671d0607e3fa14caaeacb9..89ac871d825eab79cf14cc20903b7c1dec9bc9af 100644 --- a/lib/avx_t1/crc32_sctp_avx.asm +++ b/lib/avx_t1/crc32_sctp_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/crc32_wimax_avx.asm b/lib/avx_t1/crc32_wimax_avx.asm index e8e96329a9ea4a55081fe34eb968928001e6b76a..444c8800451f54fd66ac6e1bdd8856860ce380f4 100644 --- a/lib/avx_t1/crc32_wimax_avx.asm +++ b/lib/avx_t1/crc32_wimax_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/ethernet_fcs_avx.asm b/lib/avx_t1/ethernet_fcs_avx.asm index 2e7c8e5f30453a28aedf179e179236f38a2c7a86..88632f24c955bfcd59347f941e5c43f0feefeea3 100644 --- a/lib/avx_t1/ethernet_fcs_avx.asm +++ b/lib/avx_t1/ethernet_fcs_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_refl_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/gcm_avx_gen2.asm b/lib/avx_t1/gcm_avx_gen2.asm deleted file mode 100644 index db90c7b60bdb31714b29478e718136d1eeea858d..0000000000000000000000000000000000000000 --- a/lib/avx_t1/gcm_avx_gen2.asm +++ /dev/null @@ -1,3509 +0,0 @@ -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2011-2022 Intel Corporation All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions -; are met: -; * Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; * Redistributions in binary form must reproduce the above copyright -; notice, this list of conditions and the following disclaimer in -; the documentation and/or other materials provided with the -; distribution. -; * Neither the name of Intel Corporation nor the names of its -; contributors may be used to endorse or promote products derived -; from this software without specific prior written permission. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -; -; Authors: -; Erdinc Ozturk -; Vinodh Gopal -; James Guilford -; -; -; References: -; This code was derived and highly optimized from the code described in paper: -; Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on Intel Architecture Processors. August, 2010 -; -; For the shift-based reductions used in this code, we used the method described in paper: -; Shay Gueron, Michael E. Kounavis. Intel Carry-Less Multiplication Instruction and its Usage for Computing the GCM Mode. January, 2010. -; -; -; -; -; Assumptions: -; -; -; -; iv: -; 0 1 2 3 -; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | Salt (From the SA) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | Initialization Vector | -; | (This is the sequence number from IPSec header) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 0x1 | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; -; -; -; AAD: -; AAD will be padded with 0 to the next 16byte multiple -; for example, assume AAD is a u32 vector -; -; if AAD is 8 bytes: -; AAD[3] = {A0, A1}; -; padded AAD in xmm register = {A1 A0 0 0} -; -; 0 1 2 3 -; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | SPI (A1) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 32-bit Sequence Number (A0) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 0x0 | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; -; AAD Format with 32-bit Sequence Number -; -; if AAD is 12 bytes: -; AAD[3] = {A0, A1, A2}; -; padded AAD in xmm register = {A2 A1 A0 0} -; -; 0 1 2 3 -; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | SPI (A2) | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 64-bit Extended Sequence Number {A1,A0} | -; | | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; | 0x0 | -; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -; -; AAD Format with 64-bit Extended Sequence Number -; -; -; aadLen: -; Must be a multiple of 4 bytes and from the definition of the spec. -; The code additionally supports any aadLen length. -; -; TLen: -; from the definition of the spec, TLen can only be 8, 12 or 16 bytes. -; -; poly = x^128 + x^127 + x^126 + x^121 + 1 -; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part. -; - -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" -%include "include/gcm_defines.asm" -%include "include/gcm_keys_sse_avx.asm" -%include "include/gcm_common.inc" -%include "include/memcpy.asm" -%include "include/cet.inc" -%include "include/error.inc" -%ifndef GCM128_MODE -%ifndef GCM192_MODE -%ifndef GCM256_MODE -%error "No GCM mode selected for gcm_avx_gen2.asm!" -%endif -%endif -%endif - -%ifdef GCM128_MODE -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ avx_gen2 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _128_ %+ avx_gen2 -%define NROUNDS 9 -%endif - -%ifdef GCM192_MODE -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ avx_gen2 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _192_ %+ avx_gen2 -%define NROUNDS 11 -%endif - -%ifdef GCM256_MODE -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ avx_gen2 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _256_ %+ avx_gen2 -%define NROUNDS 13 -%endif - -default rel -; need to push 4 registers into stack to maintain -%define STACK_OFFSET 8*4 - -%define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register) -%define TMP3 16*1 ; Temporary storage for AES State 3 -%define TMP4 16*2 ; Temporary storage for AES State 4 -%define TMP5 16*3 ; Temporary storage for AES State 5 -%define TMP6 16*4 ; Temporary storage for AES State 6 -%define TMP7 16*5 ; Temporary storage for AES State 7 -%define TMP8 16*6 ; Temporary storage for AES State 8 - -%define LOCAL_STORAGE 16*7 - -%ifidn __OUTPUT_FORMAT__, win64 - %define XMM_STORAGE 16*10 -%else - %define XMM_STORAGE 0 -%endif - -%define VARIABLE_OFFSET LOCAL_STORAGE + XMM_STORAGE - -mksection .text -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Utility Macros -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) -; Input: A and B (128-bits each, bit-reflected) -; Output: C = A*B*x mod poly, (i.e. >>1 ) -; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input -; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GHASH_MUL 7 -%define %%GH %1 ; 16 Bytes -%define %%HK %2 ; 16 Bytes -%define %%T1 %3 -%define %%T2 %4 -%define %%T3 %5 -%define %%T4 %6 -%define %%T5 %7 - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;; Karatsuba - vpshufd %%T2, %%GH, 01001110b - vpshufd %%T3, %%HK, 01001110b - vpxor %%T2, %%T2, %%GH ; %%T2 = (a1+a0) - vpxor %%T3, %%T3, %%HK ; %%T3 = (b1+b0) - - vpclmulqdq %%T1, %%GH, %%HK, 0x11 ; %%T1 = a1*b1 - vpclmulqdq %%GH, %%HK, 0x00 ; %%GH = a0*b0 - vpclmulqdq %%T2, %%T3, 0x00 ; %%T2 = (a1+a0)*(b1+b0) - vpxor %%T2, %%T2, %%GH - vpxor %%T2, %%T2, %%T1 ; %%T2 = a0*b1+a1*b0 - - vpslldq %%T3, %%T2, 8 ; shift-L %%T3 2 DWs - vpsrldq %%T2, %%T2, 8 ; shift-R %%T2 2 DWs - vpxor %%GH, %%GH, %%T3 - vpxor %%T1, %%T1, %%T2 ; <%%T1:%%GH> = %%GH x %%HK - - ;first phase of the reduction - vpslld %%T2, %%GH, 31 ; packed right shifting << 31 - vpslld %%T3, %%GH, 30 ; packed right shifting shift << 30 - vpslld %%T4, %%GH, 25 ; packed right shifting shift << 25 - - vpxor %%T2, %%T2, %%T3 ; xor the shifted versions - vpxor %%T2, %%T2, %%T4 - - vpsrldq %%T5, %%T2, 4 ; shift-R %%T5 1 DW - - vpslldq %%T2, %%T2, 12 ; shift-L %%T2 3 DWs - vpxor %%GH, %%GH, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - ;second phase of the reduction - - vpsrld %%T2,%%GH,1 ; packed left shifting >> 1 - vpsrld %%T3,%%GH,2 ; packed left shifting >> 2 - vpsrld %%T4,%%GH,7 ; packed left shifting >> 7 - vpxor %%T2, %%T2, %%T3 ; xor the shifted versions - vpxor %%T2, %%T2, %%T4 - - vpxor %%T2, %%T2, %%T5 - vpxor %%GH, %%GH, %%T2 - vpxor %%GH, %%GH, %%T1 ; the result is in %%GH - -%endmacro - -%macro PRECOMPUTE 8 -%define %%GDATA %1 -%define %%HK %2 -%define %%T1 %3 -%define %%T2 %4 -%define %%T3 %5 -%define %%T4 %6 -%define %%T5 %7 -%define %%T6 %8 - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Haskey_i_k holds XORed values of the low and high parts of the Haskey_i - vmovdqa %%T5, %%HK - - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_k], %%T1 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^2<<1 mod poly - vmovdqu [%%GDATA + HashKey_2], %%T5 ; [HashKey_2] = HashKey^2<<1 mod poly - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_2_k], %%T1 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^3<<1 mod poly - vmovdqu [%%GDATA + HashKey_3], %%T5 - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_3_k], %%T1 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^4<<1 mod poly - vmovdqu [%%GDATA + HashKey_4], %%T5 - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_4_k], %%T1 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^5<<1 mod poly - vmovdqu [%%GDATA + HashKey_5], %%T5 - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_5_k], %%T1 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^6<<1 mod poly - vmovdqu [%%GDATA + HashKey_6], %%T5 - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_6_k], %%T1 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^7<<1 mod poly - vmovdqu [%%GDATA + HashKey_7], %%T5 - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_7_k], %%T1 - - GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^8<<1 mod poly - vmovdqu [%%GDATA + HashKey_8], %%T5 - vpshufd %%T1, %%T5, 01001110b - vpxor %%T1, %%T5 - vmovdqu [%%GDATA + HashKey_8_k], %%T1 -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. -; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). -; Output: The hash of the data (AAD_HASH). -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro CALC_AAD_HASH 15 -%define %%A_IN %1 -%define %%A_LEN %2 -%define %%AAD_HASH %3 -%define %%GDATA_KEY %4 -%define %%XTMP0 %5 ; xmm temp reg 5 -%define %%XTMP1 %6 ; xmm temp reg 5 -%define %%XTMP2 %7 -%define %%XTMP3 %8 -%define %%XTMP4 %9 -%define %%XTMP5 %10 ; xmm temp reg 5 -%define %%T1 %11 ; temp reg 1 -%define %%T2 %12 -%define %%T3 %13 -%define %%T4 %14 -%define %%T5 %15 ; temp reg 5 - - mov %%T1, %%A_IN ; T1 = AAD - mov %%T2, %%A_LEN ; T2 = aadLen - -%%_get_AAD_loop128: - cmp %%T2, 128 - jl %%_exit_AAD_loop128 - - vmovdqu %%XTMP0, [%%T1 + 16*0] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vpxor %%XTMP0, %%AAD_HASH - - vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_8] - vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 - vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 - vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 - -%assign i 1 -%assign j 7 -%rep 7 - vmovdqu %%XTMP0, [%%T1 + 16*i] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_ %+ j] - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 - vpxor %%XTMP1, %%XTMP1, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 -%assign i (i + 1) -%assign j (j - 1) -%endrep - - vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs - vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%XTMP5, [rel POLY2] - vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 - vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;second phase of the reduction - vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 - vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 - vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 - - sub %%T2, 128 - je %%_CALC_AAD_done - - add %%T1, 128 - jmp %%_get_AAD_loop128 - -%%_exit_AAD_loop128: - cmp %%T2, 16 - jl %%_get_small_AAD_block - - ;; calculate hash_key position to start with - mov %%T3, %%T2 - and %%T3, -16 ; 1 to 7 blocks possible here - neg %%T3 - add %%T3, HashKey_1 + 16 - lea %%T3, [%%GDATA_KEY + %%T3] - - vmovdqu %%XTMP0, [%%T1] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vpxor %%XTMP0, %%AAD_HASH - - vmovdqu %%XTMP5, [%%T3] - vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 - vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 - vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 - - add %%T3, 16 ; move to next hashkey - add %%T1, 16 ; move to next data block - sub %%T2, 16 - cmp %%T2, 16 - jl %%_AAD_reduce - -%%_AAD_blocks: - vmovdqu %%XTMP0, [%%T1] - vpshufb %%XTMP0, [rel SHUF_MASK] - - vmovdqu %%XTMP5, [%%T3] - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 - vpxor %%XTMP1, %%XTMP1, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 - vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 - vpxor %%XTMP3, %%XTMP3, %%XTMP4 - - add %%T3, 16 ; move to next hashkey - add %%T1, 16 - sub %%T2, 16 - cmp %%T2, 16 - jl %%_AAD_reduce - jmp %%_AAD_blocks - -%%_AAD_reduce: - vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs - vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP4 - vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqa %%XTMP5, [rel POLY2] - vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 - vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs - vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;second phase of the reduction - vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 - vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) - - vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 - vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) - - vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 - -%%_get_small_AAD_block: - or %%T2, %%T2 - je %%_CALC_AAD_done - - vmovdqu %%XTMP0, [%%GDATA_KEY + HashKey] - READ_SMALL_DATA_INPUT_AVX %%XTMP1, %%T1, %%T2, %%T3, %%T4, %%T5 - ;byte-reflect the AAD data - vpshufb %%XTMP1, [rel SHUF_MASK] - vpxor %%AAD_HASH, %%XTMP1 - GHASH_MUL %%AAD_HASH, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 - -%%_CALC_AAD_done: - -%endmacro ; CALC_AAD_HASH - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks between update calls. -; Requires the input data be at least 1 byte long. -; Input: -; GDATA_KEY - struct gcm_key_data * -; GDATA_CTX - struct gcm_context_data * -; PLAIN_CYPH_IN - input text -; PLAIN_CYPH_LEN - input text length -; DATA_OFFSET - the current data offset -; HASH_SUBKEY - the hash subkey -; ENC_DEC - whether encoding or decoding -; Output: A cypher of the first partial block (CYPH_PLAIN_OUT), and updated GDATA_CTX -; Clobbers rax, r10, r12, r13, r15, xmm0, xmm1, xmm2, xmm3, xmm5, xmm6, xmm9, xmm10, xmm11 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro PARTIAL_BLOCK 8 -%define %%GDATA_CTX %1 -%define %%CYPH_PLAIN_OUT %2 -%define %%PLAIN_CYPH_IN %3 -%define %%PLAIN_CYPH_LEN %4 -%define %%DATA_OFFSET %5 -%define %%AAD_HASH %6 -%define %%HASH_SUBKEY %7 -%define %%ENC_DEC %8 - mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 - je %%_partial_block_done ;Leave Macro if no partial blocks - - cmp %%PLAIN_CYPH_LEN, 16 ;Read in input data without over reading - jl %%_fewer_than_16_bytes - VXLDR xmm1, [%%PLAIN_CYPH_IN] ;If more than 16 bytes of data, just fill the xmm register - jmp %%_data_read - -%%_fewer_than_16_bytes: - lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - READ_SMALL_DATA_INPUT_AVX xmm1, r10, %%PLAIN_CYPH_LEN, rax, r12, r15 - -%%_data_read: ;Finished reading in data - - vmovdqu xmm9, [%%GDATA_CTX + PBlockEncKey] ;xmm9 = my_ctx_data.partial_block_enc_key - - lea r12, [SHIFT_MASK] - - cmp r13, rax - add r12, r13 ; adjust the shuffle mask pointer to be able to shift r13 bytes (16-r13 is the number of bytes in plaintext mod 16) - vmovdqu xmm2, [r12] ; get the appropriate shuffle mask - vpshufb xmm9, xmm2 ;shift right r13 bytes - -%ifidn %%ENC_DEC, DEC - vmovdqa xmm3, xmm1 - vpxor xmm9, xmm1 ; Ciphertext XOR E(K, Yn) - - mov r15, %%PLAIN_CYPH_LEN - add r15, r13 - sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block - jge %%_no_extra_mask_1 ;Determine if if partial block is not being filled and shift mask accordingly - sub r12, r15 -%%_no_extra_mask_1: - - vmovdqu xmm1, [r12 + ALL_F-SHIFT_MASK] ; get the appropriate mask to mask out bottom r13 bytes of xmm9 - vpand xmm9, xmm1 ; mask out bottom r13 bytes of xmm9 - - vpand xmm3, xmm1 - vpshufb xmm3, [SHUF_MASK] - vpshufb xmm3, xmm2 - vpxor %%AAD_HASH, xmm3 - - cmp r15,0 - jl %%_partial_incomplete_1 - - GHASH_MUL %%AAD_HASH, %%HASH_SUBKEY, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block - xor rax,rax - mov [%%GDATA_CTX + PBlockLen], rax - jmp %%_dec_done -%%_partial_incomplete_1: -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_CYPH_LEN - add [%%GDATA_CTX + PBlockLen], rax -%else - add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN -%endif -%%_dec_done: - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH - -%else - vpxor xmm9, xmm1 ; Plaintext XOR E(K, Yn) - - mov r15, %%PLAIN_CYPH_LEN - add r15, r13 - sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block - jge %%_no_extra_mask_2 ;Determine if if partial block is not being filled and shift mask accordingly - sub r12, r15 -%%_no_extra_mask_2: - - vmovdqu xmm1, [r12 + ALL_F-SHIFT_MASK] ; get the appropriate mask to mask out bottom r13 bytes of xmm9 - vpand xmm9, xmm1 ; mask out bottom r13 bytes of xmm9 - - vpshufb xmm9, [SHUF_MASK] - vpshufb xmm9, xmm2 - vpxor %%AAD_HASH, xmm9 - - cmp r15,0 - jl %%_partial_incomplete_2 - - GHASH_MUL %%AAD_HASH, %%HASH_SUBKEY, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block - xor rax,rax - mov [%%GDATA_CTX + PBlockLen], rax - jmp %%_encode_done -%%_partial_incomplete_2: -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_CYPH_LEN - add [%%GDATA_CTX + PBlockLen], rax -%else - add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN -%endif -%%_encode_done: - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH - - vpshufb xmm9, [SHUF_MASK] ; shuffle xmm9 back to output as ciphertext - vpshufb xmm9, xmm2 -%endif - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ; output encrypted Bytes - cmp r15,0 - jl %%_partial_fill - mov r12, r13 - mov r13, 16 - sub r13, r12 ; Set r13 to be the number of bytes to write out - jmp %%_count_set -%%_partial_fill: - mov r13, %%PLAIN_CYPH_LEN -%%_count_set: - vmovq rax, xmm9 - cmp r13, 8 - jle %%_less_than_8_bytes_left - - mov [%%CYPH_PLAIN_OUT+ %%DATA_OFFSET], rax - add %%DATA_OFFSET, 8 - vpsrldq xmm9, xmm9, 8 - vmovq rax, xmm9 - sub r13, 8 -%%_less_than_8_bytes_left: - mov BYTE [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], al - add %%DATA_OFFSET, 1 - shr rax, 8 - sub r13, 1 - jne %%_less_than_8_bytes_left - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%%_partial_block_done: -%endmacro ; PARTIAL_BLOCK - -; if a = number of total plaintext bytes -; b = floor(a/16) -; %%num_initial_blocks = b mod 8; -; encrypt the initial %%num_initial_blocks blocks and apply ghash on the ciphertext -; %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r14 are used as a pointer only, not modified. -; Updated AAD_HASH is returned in %%T3 - -%macro INITIAL_BLOCKS 24 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%CYPH_PLAIN_OUT %3 -%define %%PLAIN_CYPH_IN %4 -%define %%LENGTH %5 -%define %%DATA_OFFSET %6 -%define %%num_initial_blocks %7 ; can be 0, 1, 2, 3, 4, 5, 6 or 7 -%define %%T1 %8 -%define %%HASH_KEY %9 -%define %%T3 %10 -%define %%T4 %11 -%define %%T5 %12 -%define %%CTR %13 -%define %%XMM1 %14 -%define %%XMM2 %15 -%define %%XMM3 %16 -%define %%XMM4 %17 -%define %%XMM5 %18 -%define %%XMM6 %19 -%define %%XMM7 %20 -%define %%XMM8 %21 -%define %%T6 %22 -%define %%T_key %23 -%define %%ENC_DEC %24 - -%assign i (8-%%num_initial_blocks) - vmovdqu reg(i), %%XMM8 ; move AAD_HASH to temp reg - ; start AES for %%num_initial_blocks blocks - vmovdqu %%CTR, [%%GDATA_CTX + CurCount] ; %%CTR = Y0 - -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa reg(i), %%CTR - vpshufb reg(i), [SHUF_MASK] ; perform a 16Byte swap -%assign i (i+1) -%endrep - - vmovdqu %%T_key, [%%GDATA_KEY+16*0] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vpxor reg(i),%%T_key -%assign i (i+1) -%endrep - -%assign j 1 -%rep NROUNDS - vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vaesenc reg(i),%%T_key -%assign i (i+1) -%endrep - -%assign j (j+1) -%endrep ; NROUNDS - -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - vaesenclast reg(i),%%T_key -%assign i (i+1) -%endrep - -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - vpxor reg(i), %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) ; write back ciphertext for %%num_initial_blocks blocks - add %%DATA_OFFSET, 16 - %ifidn %%ENC_DEC, DEC - vmovdqa reg(i), %%T1 - %endif - vpshufb reg(i), [SHUF_MASK] ; prepare ciphertext for GHASH computations -%assign i (i+1) -%endrep - -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) - -%rep %%num_initial_blocks - vpxor reg(j), reg(i) - GHASH_MUL reg(j), %%HASH_KEY, %%T1, %%T3, %%T4, %%T5, %%T6 ; apply GHASH on %%num_initial_blocks blocks -%assign i (i+1) -%assign j (j+1) -%endrep - ; %%XMM8 has the current Hash Value - vmovdqa %%T3, %%XMM8 - - cmp %%LENGTH, 128 - jl %%_initial_blocks_done - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Prepare 8 counter blocks and perform rounds of AES cipher on them, load plain/cipher text and -; store cipher/plain text. -; Keep 8 cipher text blocks for further GHASH computations (XMM1 - XMM8) -; - combine current GHASH value into block 0 (XMM1) - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM1, %%CTR - vpshufb %%XMM1, [SHUF_MASK] ; perform a 16Byte swap - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM2, %%CTR - vpshufb %%XMM2, [SHUF_MASK] ; perform a 16Byte swap - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM3, %%CTR - vpshufb %%XMM3, [SHUF_MASK] ; perform a 16Byte swap - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM4, %%CTR - vpshufb %%XMM4, [SHUF_MASK] ; perform a 16Byte swap - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM5, %%CTR - vpshufb %%XMM5, [SHUF_MASK] ; perform a 16Byte swap - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM6, %%CTR - vpshufb %%XMM6, [SHUF_MASK] ; perform a 16Byte swap - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM7, %%CTR - vpshufb %%XMM7, [SHUF_MASK] ; perform a 16Byte swap - - vpaddd %%CTR, [ONE] ; INCR Y0 - vmovdqa %%XMM8, %%CTR - vpshufb %%XMM8, [SHUF_MASK] ; perform a 16Byte swap - - vmovdqu %%T_key, [%%GDATA_KEY+16*0] - vpxor %%XMM1, %%T_key - vpxor %%XMM2, %%T_key - vpxor %%XMM3, %%T_key - vpxor %%XMM4, %%T_key - vpxor %%XMM5, %%T_key - vpxor %%XMM6, %%T_key - vpxor %%XMM7, %%T_key - vpxor %%XMM8, %%T_key - -%assign i 1 -%rep NROUNDS - vmovdqu %%T_key, [%%GDATA_KEY+16*i] - vaesenc %%XMM1, %%T_key - vaesenc %%XMM2, %%T_key - vaesenc %%XMM3, %%T_key - vaesenc %%XMM4, %%T_key - vaesenc %%XMM5, %%T_key - vaesenc %%XMM6, %%T_key - vaesenc %%XMM7, %%T_key - vaesenc %%XMM8, %%T_key -%assign i (i+1) -%endrep - - vmovdqu %%T_key, [%%GDATA_KEY+16*i] - vaesenclast %%XMM1, %%T_key - vaesenclast %%XMM2, %%T_key - vaesenclast %%XMM3, %%T_key - vaesenclast %%XMM4, %%T_key - vaesenclast %%XMM5, %%T_key - vaesenclast %%XMM6, %%T_key - vaesenclast %%XMM7, %%T_key - vaesenclast %%XMM8, %%T_key - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*0] - vpxor %%XMM1, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*0], %%XMM1 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM1, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*1] - vpxor %%XMM2, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*1], %%XMM2 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM2, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*2] - vpxor %%XMM3, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*2], %%XMM3 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM3, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*3] - vpxor %%XMM4, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*3], %%XMM4 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM4, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*4] - vpxor %%XMM5, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*4], %%XMM5 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM5, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*5] - vpxor %%XMM6, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*5], %%XMM6 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM6, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*6] - vpxor %%XMM7, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*6], %%XMM7 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM7, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*7] - vpxor %%XMM8, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*7], %%XMM8 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM8, %%T1 - %endif - - add %%DATA_OFFSET, 128 - - vpshufb %%XMM1, [SHUF_MASK] ; perform a 16Byte swap - vpxor %%XMM1, %%T3 ; combine GHASHed value with the corresponding ciphertext - vpshufb %%XMM2, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM3, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM4, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM5, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM6, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM7, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM8, [SHUF_MASK] ; perform a 16Byte swap - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%%_initial_blocks_done: - -%endmacro - -; encrypt 8 blocks at a time -; ghash the 8 previously encrypted ciphertext blocks -; %%GDATA - (GCM key data), %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN are used as pointers only, not modified -; r11 is the data offset value -%macro GHASH_8_ENCRYPT_8_PARALLEL 22 -%define %%GDATA %1 -%define %%CYPH_PLAIN_OUT %2 -%define %%PLAIN_CYPH_IN %3 -%define %%DATA_OFFSET %4 -%define %%T1 %5 -%define %%T2 %6 -%define %%T3 %7 -%define %%T4 %8 -%define %%T5 %9 -%define %%T6 %10 -%define %%CTR %11 -%define %%XMM1 %12 -%define %%XMM2 %13 -%define %%XMM3 %14 -%define %%XMM4 %15 -%define %%XMM5 %16 -%define %%XMM6 %17 -%define %%XMM7 %18 -%define %%XMM8 %19 -%define %%T7 %20 -%define %%loop_idx %21 -%define %%ENC_DEC %22 - - vmovdqa %%T2, %%XMM1 - vmovdqu [rsp + TMP2], %%XMM2 - vmovdqu [rsp + TMP3], %%XMM3 - vmovdqu [rsp + TMP4], %%XMM4 - vmovdqu [rsp + TMP5], %%XMM5 - vmovdqu [rsp + TMP6], %%XMM6 - vmovdqu [rsp + TMP7], %%XMM7 - vmovdqu [rsp + TMP8], %%XMM8 - -%ifidn %%loop_idx, in_order - vpaddd %%XMM1, %%CTR, [ONE] ; INCR CNT - vpaddd %%XMM2, %%XMM1, [ONE] - vpaddd %%XMM3, %%XMM2, [ONE] - vpaddd %%XMM4, %%XMM3, [ONE] - vpaddd %%XMM5, %%XMM4, [ONE] - vpaddd %%XMM6, %%XMM5, [ONE] - vpaddd %%XMM7, %%XMM6, [ONE] - vpaddd %%XMM8, %%XMM7, [ONE] - vmovdqa %%CTR, %%XMM8 - - vpshufb %%XMM1, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM2, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM3, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM4, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM5, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM6, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM7, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM8, [SHUF_MASK] ; perform a 16Byte swap -%else - vpaddd %%XMM1, %%CTR, [ONEf] ; INCR CNT - vpaddd %%XMM2, %%XMM1, [ONEf] - vpaddd %%XMM3, %%XMM2, [ONEf] - vpaddd %%XMM4, %%XMM3, [ONEf] - vpaddd %%XMM5, %%XMM4, [ONEf] - vpaddd %%XMM6, %%XMM5, [ONEf] - vpaddd %%XMM7, %%XMM6, [ONEf] - vpaddd %%XMM8, %%XMM7, [ONEf] - vmovdqa %%CTR, %%XMM8 -%endif - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T1, [%%GDATA + 16*0] - vpxor %%XMM1, %%T1 - vpxor %%XMM2, %%T1 - vpxor %%XMM3, %%T1 - vpxor %%XMM4, %%T1 - vpxor %%XMM5, %%T1 - vpxor %%XMM6, %%T1 - vpxor %%XMM7, %%T1 - vpxor %%XMM8, %%T1 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T1, [%%GDATA + 16*1] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [%%GDATA + 16*2] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + HashKey_8] - vpclmulqdq %%T4, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 - vpclmulqdq %%T7, %%T2, %%T5, 0x00 ; %%T7 = a0*b0 - - vpshufd %%T6, %%T2, 01001110b - vpxor %%T6, %%T2 - - vmovdqu %%T5, [%%GDATA + HashKey_8_k] - vpclmulqdq %%T6, %%T6, %%T5, 0x00 ; - - vmovdqu %%T1, [%%GDATA + 16*3] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP2] - vmovdqu %%T5, [%%GDATA + HashKey_7] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpshufd %%T3, %%T1, 01001110b - vpxor %%T3, %%T1 - vmovdqu %%T5, [%%GDATA + HashKey_7_k] - vpclmulqdq %%T3, %%T3, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*4] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vmovdqu %%T1, [rsp + TMP3] - vmovdqu %%T5, [%%GDATA + HashKey_6] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpshufd %%T3, %%T1, 01001110b - vpxor %%T3, %%T1 - vmovdqu %%T5, [%%GDATA + HashKey_6_k] - vpclmulqdq %%T3, %%T3, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*5] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP4] - vmovdqu %%T5, [%%GDATA + HashKey_5] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpshufd %%T3, %%T1, 01001110b - vpxor %%T3, %%T1 - vmovdqu %%T5, [%%GDATA + HashKey_5_k] - vpclmulqdq %%T3, %%T3, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*6] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP5] - vmovdqu %%T5, [%%GDATA + HashKey_4] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpshufd %%T3, %%T1, 01001110b - vpxor %%T3, %%T1 - vmovdqu %%T5, [%%GDATA + HashKey_4_k] - vpclmulqdq %%T3, %%T3, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*7] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP6] - vmovdqu %%T5, [%%GDATA + HashKey_3] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpshufd %%T3, %%T1, 01001110b - vpxor %%T3, %%T1 - vmovdqu %%T5, [%%GDATA + HashKey_3_k] - vpclmulqdq %%T3, %%T3, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vmovdqu %%T1, [%%GDATA + 16*8] - vaesenc %%XMM1, %%T1 - vaesenc %%XMM2, %%T1 - vaesenc %%XMM3, %%T1 - vaesenc %%XMM4, %%T1 - vaesenc %%XMM5, %%T1 - vaesenc %%XMM6, %%T1 - vaesenc %%XMM7, %%T1 - vaesenc %%XMM8, %%T1 - - vmovdqu %%T1, [rsp + TMP7] - vmovdqu %%T5, [%%GDATA + HashKey_2] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpshufd %%T3, %%T1, 01001110b - vpxor %%T3, %%T1 - vmovdqu %%T5, [%%GDATA + HashKey_2_k] - vpclmulqdq %%T3, %%T3, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vmovdqu %%T5, [%%GDATA + 16*9] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T1, [rsp + TMP8] - vmovdqu %%T5, [%%GDATA + HashKey] - vpclmulqdq %%T3, %%T1, %%T5, 0x11 - vpxor %%T4, %%T4, %%T3 - vpclmulqdq %%T3, %%T1, %%T5, 0x00 - vpxor %%T7, %%T7, %%T3 - - vpshufd %%T3, %%T1, 01001110b - vpxor %%T3, %%T1 - vmovdqu %%T5, [%%GDATA + HashKey_k] - vpclmulqdq %%T3, %%T3, %%T5, 0x10 - vpxor %%T6, %%T6, %%T3 - - vpxor %%T6, %%T4 - vpxor %%T6, %%T7 - -%ifdef GCM128_MODE - vmovdqu %%T5, [%%GDATA + 16*10] -%endif -%ifdef GCM192_MODE - vmovdqu %%T5, [%%GDATA + 16*10] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*11] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*12] -%endif -%ifdef GCM256_MODE - vmovdqu %%T5, [%%GDATA + 16*10] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*11] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*12] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*13] - vaesenc %%XMM1, %%T5 - vaesenc %%XMM2, %%T5 - vaesenc %%XMM3, %%T5 - vaesenc %%XMM4, %%T5 - vaesenc %%XMM5, %%T5 - vaesenc %%XMM6, %%T5 - vaesenc %%XMM7, %%T5 - vaesenc %%XMM8, %%T5 - - vmovdqu %%T5, [%%GDATA + 16*14] -%endif - -%assign i 0 -%assign j 1 -%rep 8 - -%ifidn %%ENC_DEC, ENC -%ifdef NT_LD - VXLDR %%T2, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*i] - vpxor %%T2, %%T2, %%T5 -%else - vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*i] -%endif ; NT_LD - vaesenclast reg(j), reg(j), %%T2 -%else - VXLDR %%T2, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*i] - vpxor %%T2, %%T2, %%T5 - vaesenclast %%T3, reg(j), %%T2 - vpxor reg(j), %%T2, %%T5 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*i], %%T3 -%endif ; %%ENC_DEC - -%assign i (i+1) -%assign j (j+1) -%endrep - - vpslldq %%T3, %%T6, 8 ; shift-L %%T3 2 DWs - vpsrldq %%T6, %%T6, 8 ; shift-R %%T2 2 DWs - vpxor %%T7, %%T3 - vpxor %%T6, %%T4 ; accumulate the results in %%T6:%%T7 - - ;first phase of the reduction - - vpslld %%T2, %%T7, 31 ; packed right shifting << 31 - vpslld %%T3, %%T7, 30 ; packed right shifting shift << 30 - vpslld %%T4, %%T7, 25 ; packed right shifting shift << 25 - - vpxor %%T2, %%T2, %%T3 ; xor the shifted versions - vpxor %%T2, %%T2, %%T4 - - vpsrldq %%T1, %%T2, 4 ; shift-R %%T1 1 DW - - vpslldq %%T2, %%T2, 12 ; shift-L %%T2 3 DWs - vpxor %%T7, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - %ifidn %%ENC_DEC, ENC - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*0], %%XMM1 ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*1], %%XMM2 ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*2], %%XMM3 ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*3], %%XMM4 ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*4], %%XMM5 ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*5], %%XMM6 ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*6], %%XMM7 ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*7], %%XMM8 ; Write to the Ciphertext buffer - %endif - - ;second phase of the reduction - - vpsrld %%T2,%%T7,1 ; packed left shifting >> 1 - vpsrld %%T3,%%T7,2 ; packed left shifting >> 2 - vpsrld %%T4,%%T7,7 ; packed left shifting >> 7 - vpxor %%T2, %%T2,%%T3 ; xor the shifted versions - vpxor %%T2, %%T2,%%T4 - - vpxor %%T2, %%T2, %%T1 - vpxor %%T7, %%T7, %%T2 - vpxor %%T6, %%T6, %%T7 ; the result is in %%T6 - - vpshufb %%XMM1, [SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM2, [SHUF_MASK] - vpshufb %%XMM3, [SHUF_MASK] - vpshufb %%XMM4, [SHUF_MASK] - vpshufb %%XMM5, [SHUF_MASK] - vpshufb %%XMM6, [SHUF_MASK] - vpshufb %%XMM7, [SHUF_MASK] - vpshufb %%XMM8, [SHUF_MASK] - - vpxor %%XMM1, %%T6 - -%endmacro - -; GHASH the last 4 ciphertext blocks. -; %%GDATA is GCM key data -%macro GHASH_LAST_8 16 -%define %%GDATA %1 -%define %%T1 %2 -%define %%T2 %3 -%define %%T3 %4 -%define %%T4 %5 -%define %%T5 %6 -%define %%T6 %7 -%define %%T7 %8 -%define %%XMM1 %9 -%define %%XMM2 %10 -%define %%XMM3 %11 -%define %%XMM4 %12 -%define %%XMM5 %13 -%define %%XMM6 %14 -%define %%XMM7 %15 -%define %%XMM8 %16 - ;; Karatsuba Method - - vpshufd %%T2, %%XMM1, 01001110b - vpxor %%T2, %%XMM1 - vmovdqu %%T5, [%%GDATA + HashKey_8] - vpclmulqdq %%T6, %%XMM1, %%T5, 0x11 - vpclmulqdq %%T7, %%XMM1, %%T5, 0x00 - - vmovdqu %%T3, [%%GDATA + HashKey_8_k] - vpclmulqdq %%XMM1, %%T2, %%T3, 0x00 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpshufd %%T2, %%XMM2, 01001110b - vpxor %%T2, %%XMM2 - vmovdqu %%T5, [%%GDATA + HashKey_7] - vpclmulqdq %%T4, %%XMM2, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM2, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vmovdqu %%T3, [%%GDATA + HashKey_7_k] - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpshufd %%T2, %%XMM3, 01001110b - vpxor %%T2, %%XMM3 - vmovdqu %%T5, [%%GDATA + HashKey_6] - vpclmulqdq %%T4, %%XMM3, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM3, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vmovdqu %%T3, [%%GDATA + HashKey_6_k] - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpshufd %%T2, %%XMM4, 01001110b - vpxor %%T2, %%XMM4 - vmovdqu %%T5, [%%GDATA + HashKey_5] - vpclmulqdq %%T4, %%XMM4, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM4, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vmovdqu %%T3, [%%GDATA + HashKey_5_k] - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpshufd %%T2, %%XMM5, 01001110b - vpxor %%T2, %%XMM5 - vmovdqu %%T5, [%%GDATA + HashKey_4] - vpclmulqdq %%T4, %%XMM5, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM5, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vmovdqu %%T3, [%%GDATA + HashKey_4_k] - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpshufd %%T2, %%XMM6, 01001110b - vpxor %%T2, %%XMM6 - vmovdqu %%T5, [%%GDATA + HashKey_3] - - vpclmulqdq %%T4, %%XMM6, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM6, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vmovdqu %%T3, [%%GDATA + HashKey_3_k] - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpshufd %%T2, %%XMM7, 01001110b - vpxor %%T2, %%XMM7 - vmovdqu %%T5, [%%GDATA + HashKey_2] - vpclmulqdq %%T4, %%XMM7, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM7, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vmovdqu %%T3, [%%GDATA + HashKey_2_k] - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - vpxor %%XMM1, %%XMM1, %%T2 - - ;;;;;;;;;;;;;;;;;;;;;; - - vpshufd %%T2, %%XMM8, 01001110b - vpxor %%T2, %%XMM8 - vmovdqu %%T5, [%%GDATA + HashKey] - vpclmulqdq %%T4, %%XMM8, %%T5, 0x11 - vpxor %%T6, %%T6, %%T4 - - vpclmulqdq %%T4, %%XMM8, %%T5, 0x00 - vpxor %%T7, %%T7, %%T4 - - vmovdqu %%T3, [%%GDATA + HashKey_k] - vpclmulqdq %%T2, %%T2, %%T3, 0x00 - - vpxor %%XMM1, %%XMM1, %%T2 - vpxor %%XMM1, %%XMM1, %%T6 - vpxor %%T2, %%XMM1, %%T7 - - vpslldq %%T4, %%T2, 8 - vpsrldq %%T2, %%T2, 8 - - vpxor %%T7, %%T4 - vpxor %%T6, %%T2 ; <%%T6:%%T7> holds the result of the accumulated carry-less multiplications - - ;first phase of the reduction - - vpslld %%T2, %%T7, 31 ; packed right shifting << 31 - vpslld %%T3, %%T7, 30 ; packed right shifting shift << 30 - vpslld %%T4, %%T7, 25 ; packed right shifting shift << 25 - - vpxor %%T2, %%T2, %%T3 ; xor the shifted versions - vpxor %%T2, %%T2, %%T4 - - vpsrldq %%T1, %%T2, 4 ; shift-R %%T1 1 DW - - vpslldq %%T2, %%T2, 12 ; shift-L %%T2 3 DWs - vpxor %%T7, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - ;second phase of the reduction - - vpsrld %%T2,%%T7,1 ; packed left shifting >> 1 - vpsrld %%T3,%%T7,2 ; packed left shifting >> 2 - vpsrld %%T4,%%T7,7 ; packed left shifting >> 7 - vpxor %%T2, %%T2,%%T3 ; xor the shifted versions - vpxor %%T2, %%T2,%%T4 - - vpxor %%T2, %%T2, %%T1 - vpxor %%T7, %%T7, %%T2 - vpxor %%T6, %%T6, %%T7 ; the result is in %%T6 - -%endmacro - -; Encryption of a single block -; %%GDATA is GCM key data -%macro ENCRYPT_SINGLE_BLOCK 2 -%define %%GDATA %1 -%define %%XMM0 %2 - - vpxor %%XMM0, [%%GDATA+16*0] -%assign i 1 -%rep NROUNDS - vaesenc %%XMM0, [%%GDATA+16*i] -%assign i (i+1) -%endrep ; NROUNDS - vaesenclast %%XMM0, [%%GDATA+16*i] -%endmacro - -;; Start of Stack Setup - -%macro FUNC_SAVE 0 - ;; Required for Update/GMC_ENC - ;the number of pushes must equal STACK_OFFSET - push r12 - push r13 - push r14 - push r15 - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 - -%ifidn __OUTPUT_FORMAT__, win64 - ; xmm6:xmm15 need to be maintained for Windows - vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 - vmovdqu [rsp + LOCAL_STORAGE + 1*16],xmm7 - vmovdqu [rsp + LOCAL_STORAGE + 2*16],xmm8 - vmovdqu [rsp + LOCAL_STORAGE + 3*16],xmm9 - vmovdqu [rsp + LOCAL_STORAGE + 4*16],xmm10 - vmovdqu [rsp + LOCAL_STORAGE + 5*16],xmm11 - vmovdqu [rsp + LOCAL_STORAGE + 6*16],xmm12 - vmovdqu [rsp + LOCAL_STORAGE + 7*16],xmm13 - vmovdqu [rsp + LOCAL_STORAGE + 8*16],xmm14 - vmovdqu [rsp + LOCAL_STORAGE + 9*16],xmm15 -%endif -%endmacro - -%macro FUNC_RESTORE 0 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_xmms_avx_asm -%endif -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm15 , [rsp + LOCAL_STORAGE + 9*16] - vmovdqu xmm14 , [rsp + LOCAL_STORAGE + 8*16] - vmovdqu xmm13 , [rsp + LOCAL_STORAGE + 7*16] - vmovdqu xmm12 , [rsp + LOCAL_STORAGE + 6*16] - vmovdqu xmm11 , [rsp + LOCAL_STORAGE + 5*16] - vmovdqu xmm10 , [rsp + LOCAL_STORAGE + 4*16] - vmovdqu xmm9 , [rsp + LOCAL_STORAGE + 3*16] - vmovdqu xmm8 , [rsp + LOCAL_STORAGE + 2*16] - vmovdqu xmm7 , [rsp + LOCAL_STORAGE + 1*16] - vmovdqu xmm6 , [rsp + LOCAL_STORAGE + 0*16] -%endif - -;; Required for Update/GMC_ENC - mov rsp, r14 - pop r15 - pop r14 - pop r13 - pop r12 -%endmacro - -%macro CALC_J0 15 -%define %%KEY %1 ;; [in] Pointer to GCM KEY structure -%define %%IV %2 ;; [in] Pointer to IV -%define %%IV_LEN %3 ;; [in] IV length -%define %%J0 %4 ;; [out] XMM reg to contain J0 -%define %%TMP0 %5 ;; [clobbered] Temporary GP reg -%define %%TMP1 %6 ;; [clobbered] Temporary GP reg -%define %%TMP2 %7 ;; [clobbered] Temporary GP reg -%define %%TMP3 %8 ;; [clobbered] Temporary GP reg -%define %%TMP4 %9 ;; [clobbered] Temporary GP reg -%define %%XTMP0 %10 ;; [clobbered] Temporary XMM reg -%define %%XTMP1 %11 ;; [clobbered] Temporary XMM reg -%define %%XTMP2 %12 ;; [clobbered] Temporary XMM reg -%define %%XTMP3 %13 ;; [clobbered] Temporary XMM reg -%define %%XTMP4 %14 ;; [clobbered] Temporary XMM reg -%define %%XTMP5 %15 ;; [clobbered] Temporary XMM reg - - ;; J0 = GHASH(IV || 0s+64 || len(IV)64) - ;; s = 16 * RoundUp(len(IV)/16) - len(IV) */ - - ;; Calculate GHASH of (IV || 0s) - vpxor %%J0, %%J0 - CALC_AAD_HASH %%IV, %%IV_LEN, %%J0, %%KEY, %%XTMP0, %%XTMP1, %%XTMP2, \ - %%XTMP3, %%XTMP4, %%XTMP5, %%TMP0, %%TMP1, %%TMP2, %%TMP3, %%TMP4 - - ;; Calculate GHASH of last 16-byte block (0 || len(IV)64) - vmovdqu %%XTMP0, [%%KEY + HashKey] - mov %%TMP2, %%IV_LEN - shl %%TMP2, 3 ;; IV length in bits - vmovq %%XTMP1, %%TMP2 - vpxor %%J0, %%XTMP1 - GHASH_MUL %%J0, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 - - vpshufb %%J0, [rel SHUF_MASK] ; perform a 16Byte swap -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GCM_INIT initializes a gcm_context_data struct to prepare for encoding/decoding. -; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), IV, IV_LEN, -; Additional Authentication data (A_IN), Additional Data length (A_LEN). -; Output: Updated GDATA_CTX with the hash of A_IN (AadHash) and initialized other parts of GDATA. -; Clobbers rax, r10-r13 and xmm0-xmm6 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GCM_INIT 5-6 -%define %%GDATA_KEY %1 ; [in] Pointer to GCM Key data structure -%define %%GDATA_CTX %2 ; [in/out] Pointer to GCM Context data structure -%define %%IV %3 ; [in] Pointer to IV -%define %%A_IN %4 ; [in] Pointer to AAD -%define %%A_LEN %5 ; [in] AAD length -%define %%IV_LEN %6 ; [in] IV length - -%define %%AAD_HASH xmm0 - - vpxor %%AAD_HASH, %%AAD_HASH - CALC_AAD_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, r10, r11, r12, r13, rax - vpxor xmm2, xmm3 - mov r10, %%A_LEN - - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH ; ctx_data.aad hash = aad_hash - mov [%%GDATA_CTX + AadLen], r10 ; ctx_data.aad_length = aad_length - xor r10, r10 - mov [%%GDATA_CTX + InLen], r10 ; ctx_data.in_length = 0 - mov [%%GDATA_CTX + PBlockLen], r10 ; ctx_data.partial_block_length = 0 - vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm2 ; ctx_data.partial_block_enc_key = 0 - mov r10, %%IV -%if %0 == 6 ;; IV is different than 12 bytes - CALC_J0 %%GDATA_KEY, %%IV, %%IV_LEN, xmm2, r10, r11, r12, r13, rax, xmm1, xmm0, \ - xmm3, xmm4, xmm5, xmm6 -%else ;; IV is 12 bytes - vmovdqa xmm2, [rel ONEf] ; read 12 IV bytes and pad with 0x00000001 - vpinsrq xmm2, [r10], 0 - vpinsrd xmm2, [r10+8], 2 -%endif - vmovdqu [%%GDATA_CTX + OrigIV], xmm2 ; ctx_data.orig_IV = iv - - vpshufb xmm2, [rel SHUF_MASK] - - vmovdqu [%%GDATA_CTX + CurCount], xmm2 ; ctx_data.current_counter = iv -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context_data struct -; has been initialized by GCM_INIT -; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. -; Input: struct gcm_key_data* (GDATA_KEY), struct gcm_context_data * (GDATA_CTX), -; input text (PLAIN_CYPH_IN), input text length (PLAIN_CYPH_LEN), -; and whether encoding or decoding (ENC_DEC) -; Output: A cypher of the given plain text (CYPH_PLAIN_OUT), and updated GDATA_CTX -; Clobbers rax, r10-r15, and xmm0-xmm15 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GCM_ENC_DEC 6 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%CYPH_PLAIN_OUT %3 -%define %%PLAIN_CYPH_IN %4 -%define %%PLAIN_CYPH_LEN %5 -%define %%ENC_DEC %6 -%define %%DATA_OFFSET r11 - -; Macro flow: -; calculate the number of 16byte blocks in the message -; process (number of 16byte blocks) mod 8 '%%_initial_num_blocks_is_# .. %%_initial_blocks_encrypted' -; process 8 16 byte blocks at a time until all are done '%%_encrypt_by_8_new .. %%_eight_cipher_left' -; if there is a block of less than 16 bytes process it '%%_zero_cipher_left .. %%_multiple_of_16_bytes' - cmp %%PLAIN_CYPH_LEN, 0 - je %%_multiple_of_16_bytes - - xor %%DATA_OFFSET, %%DATA_OFFSET -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_CYPH_LEN - add [%%GDATA_CTX + InLen], rax ; Update length of data processed -%else - add [%%GDATA_CTX + InLen], %%PLAIN_CYPH_LEN ; Update length of data processed -%endif - vmovdqu xmm13, [%%GDATA_KEY + HashKey] ; xmm13 = HashKey - vmovdqu xmm8, [%%GDATA_CTX + AadHash] - - PARTIAL_BLOCK %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%DATA_OFFSET, xmm8, xmm13, %%ENC_DEC - - mov r13, %%PLAIN_CYPH_LEN - sub r13, %%DATA_OFFSET - mov r10, r13 ; save the amount of data left to process in r10 - and r13, -16 ; r13 = r13 - (r13 mod 16) - - mov r12, r13 - shr r12, 4 - and r12, 7 - - jz %%_initial_num_blocks_is_0 - - cmp r12, 7 - je %%_initial_num_blocks_is_7 - cmp r12, 6 - je %%_initial_num_blocks_is_6 - cmp r12, 5 - je %%_initial_num_blocks_is_5 - cmp r12, 4 - je %%_initial_num_blocks_is_4 - cmp r12, 3 - je %%_initial_num_blocks_is_3 - cmp r12, 2 - je %%_initial_num_blocks_is_2 - - jmp %%_initial_num_blocks_is_1 - -%%_initial_num_blocks_is_7: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 7, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - sub r13, 16*7 - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_6: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 6, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - sub r13, 16*6 - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_5: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 5, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - sub r13, 16*5 - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_4: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 4, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - sub r13, 16*4 - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_3: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 3, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - sub r13, 16*3 - jmp %%_initial_blocks_encrypted -%%_initial_num_blocks_is_2: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 2, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - sub r13, 16*2 - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_1: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 1, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - sub r13, 16 - jmp %%_initial_blocks_encrypted - -%%_initial_num_blocks_is_0: - INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 0, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC - -%%_initial_blocks_encrypted: - cmp r13, 0 - je %%_zero_cipher_left - - sub r13, 128 - je %%_eight_cipher_left - - vmovd r15d, xmm9 - and r15d, 255 - vpshufb xmm9, [SHUF_MASK] - -%%_encrypt_by_8_new: - cmp r15d, 255-8 - jg %%_encrypt_by_8 - - add r15b, 8 - GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, out_order, %%ENC_DEC - add %%DATA_OFFSET, 128 - sub r13, 128 - jne %%_encrypt_by_8_new - - vpshufb xmm9, [SHUF_MASK] - jmp %%_eight_cipher_left - -%%_encrypt_by_8: - vpshufb xmm9, [SHUF_MASK] - add r15b, 8 - GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN,%%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, in_order, %%ENC_DEC - vpshufb xmm9, [SHUF_MASK] - add %%DATA_OFFSET, 128 - sub r13, 128 - jne %%_encrypt_by_8_new - - vpshufb xmm9, [SHUF_MASK] - -%%_eight_cipher_left: - GHASH_LAST_8 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 - -%%_zero_cipher_left: - vmovdqu [%%GDATA_CTX + AadHash], xmm14 ; ctx_data.aad hash = xmm14 - vmovdqu [%%GDATA_CTX + CurCount], xmm9 ; ctx_data.current_counter = xmm9 - - mov r13, r10 - and r13, 15 ; r13 = (%%PLAIN_CYPH_LEN mod 16) - - je %%_multiple_of_16_bytes - - mov [%%GDATA_CTX + PBlockLen], r13 ; ctx_data.partial_blck_length = r13 - ; handle the last <16 Byte block separately - - vpaddd xmm9, [ONE] ; INCR CNT to get Yn - vmovdqu [%%GDATA_CTX + CurCount], xmm9 ; my_ctx_data.current_counter = xmm9 - vpshufb xmm9, [SHUF_MASK] - ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Yn) - vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm9 ; ctx_data.partial_block_enc_key = xmm9 - - cmp %%PLAIN_CYPH_LEN, 16 - jge %%_large_enough_update - - lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - READ_SMALL_DATA_INPUT_AVX xmm1, r10, r13, r12, r15, rax - lea r12, [SHIFT_MASK + 16] - sub r12, r13 - jmp %%_data_read - -%%_large_enough_update: - sub %%DATA_OFFSET, 16 - add %%DATA_OFFSET, r13 - - vmovdqu xmm1, [%%PLAIN_CYPH_IN+%%DATA_OFFSET] ; receive the last <16 Byte block - - sub %%DATA_OFFSET, r13 - add %%DATA_OFFSET, 16 - - lea r12, [SHIFT_MASK + 16] - sub r12, r13 ; adjust the shuffle mask pointer to be able to shift 16-r13 bytes (r13 is the number of bytes in plaintext mod 16) - - vmovdqu xmm2, [r12] ; get the appropriate shuffle mask - vpshufb xmm1, xmm2 ; shift right 16-r13 bytes -%%_data_read: -%ifidn %%ENC_DEC, DEC - vmovdqa xmm2, xmm1 - vpxor xmm9, xmm1 ; Plaintext XOR E(K, Yn) - vmovdqu xmm1, [r12 + ALL_F - SHIFT_MASK] ; get the appropriate mask to mask out top 16-r13 bytes of xmm9 - vpand xmm9, xmm1 ; mask out top 16-r13 bytes of xmm9 - vpand xmm2, xmm1 - vpshufb xmm2, [SHUF_MASK] - vpxor xmm14, xmm2 - vmovdqu [%%GDATA_CTX + AadHash], xmm14 - -%else - vpxor xmm9, xmm1 ; Plaintext XOR E(K, Yn) - vmovdqu xmm1, [r12 + ALL_F - SHIFT_MASK] ; get the appropriate mask to mask out top 16-r13 bytes of xmm9 - vpand xmm9, xmm1 ; mask out top 16-r13 bytes of xmm9 - vpshufb xmm9, [SHUF_MASK] - vpxor xmm14, xmm9 - vmovdqu [%%GDATA_CTX + AadHash], xmm14 - - vpshufb xmm9, [SHUF_MASK] ; shuffle xmm9 back to output as ciphertext -%endif - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ; output r13 Bytes - vmovq rax, xmm9 - cmp r13, 8 - jle %%_less_than_8_bytes_left - - mov [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], rax - add %%DATA_OFFSET, 8 - vpsrldq xmm9, xmm9, 8 - vmovq rax, xmm9 - sub r13, 8 - -%%_less_than_8_bytes_left: - mov BYTE [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], al - add %%DATA_OFFSET, 1 - shr rax, 8 - sub r13, 1 - jne %%_less_than_8_bytes_left - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -%%_multiple_of_16_bytes: - -%endmacro - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; GCM_COMPLETE Finishes Encryption/Decryption of last partial block after GCM_UPDATE finishes. -; Input: struct gcm_key_data* (GDATA_KEY), struct gcm_context_data *(GDATA_CTX). -; Output: Authorization Tag (AUTH_TAG) and Authorization Tag length (AUTH_TAG_LEN) -; Clobbers rax, r10-r12, and xmm0-xmm2, xmm5-xmm6, xmm9-xmm11, xmm13-xmm15 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro GCM_COMPLETE 4 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%AUTH_TAG %3 -%define %%AUTH_TAG_LEN %4 -%define %%PLAIN_CYPH_LEN rax - - mov r12, [%%GDATA_CTX + PBlockLen] - vmovdqu xmm14, [%%GDATA_CTX + AadHash] - vmovdqu xmm13, [%%GDATA_KEY + HashKey] - - cmp r12, 0 - - je %%_partial_done - - GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block - vmovdqu [%%GDATA_CTX + AadHash], xmm14 - -%%_partial_done: - - mov r12, [%%GDATA_CTX + AadLen] ; r12 = aadLen (number of bytes) - mov %%PLAIN_CYPH_LEN, [%%GDATA_CTX + InLen] - - shl r12, 3 ; convert into number of bits - vmovq xmm15, r12 ; len(A) in xmm15 - - shl %%PLAIN_CYPH_LEN, 3 ; len(C) in bits (*128) - vmovq xmm1, %%PLAIN_CYPH_LEN - vpslldq xmm15, xmm15, 8 ; xmm15 = len(A)|| 0x0000000000000000 - vpxor xmm15, xmm1 ; xmm15 = len(A)||len(C) - - vpxor xmm14, xmm15 - GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ; final GHASH computation - vpshufb xmm14, [SHUF_MASK] ; perform a 16Byte swap - - vmovdqu xmm9, [%%GDATA_CTX + OrigIV] ; xmm9 = Y0 - - ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Y0) - - vpxor xmm9, xmm14 - -%%_return_T: - mov r10, %%AUTH_TAG ; r10 = authTag - mov r11, %%AUTH_TAG_LEN ; r11 = auth_tag_len - - cmp r11, 16 - je %%_T_16 - - cmp r11, 12 - je %%_T_12 - - cmp r11, 8 - je %%_T_8 - - simd_store_avx r10, xmm9, r11, r12, rax - jmp %%_return_T_done -%%_T_8: - vmovq rax, xmm9 - mov [r10], rax - jmp %%_return_T_done -%%_T_12: - vmovq rax, xmm9 - mov [r10], rax - vpsrldq xmm9, xmm9, 8 - vmovd eax, xmm9 - mov [r10 + 8], eax - jmp %%_return_T_done -%%_T_16: - vmovdqu [r10], xmm9 - -%%_return_T_done: - -%ifdef SAFE_DATA - ;; Clear sensitive data from context structure - vpxor xmm0, xmm0 - vmovdqu [%%GDATA_CTX + AadHash], xmm0 - vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm0 -%endif -%endmacro ; GCM_COMPLETE - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_precomp_128_avx_gen2 -; (struct gcm_key_data *key_data); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(precomp,_),function,) -FN_NAME(precomp,_): - endbranch64 -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_precomp -%endif - - push r12 - push r13 - push r14 - push r15 - - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 ; align rsp to 64 bytes - -%ifidn __OUTPUT_FORMAT__, win64 - ; only xmm6 needs to be maintained - vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 -%endif - - vpxor xmm6, xmm6 - ENCRYPT_SINGLE_BLOCK arg1, xmm6 ; xmm6 = HashKey - - vpshufb xmm6, [SHUF_MASK] - ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; - vmovdqa xmm2, xmm6 - vpsllq xmm6, 1 - vpsrlq xmm2, 63 - vmovdqa xmm1, xmm2 - vpslldq xmm2, xmm2, 8 - vpsrldq xmm1, xmm1, 8 - vpor xmm6, xmm2 - ;reduction - vpshufd xmm2, xmm1, 00100100b - vpcmpeqd xmm2, [TWOONE] - vpand xmm2, [POLY] - vpxor xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vmovdqu [arg1 + HashKey], xmm6 ; store HashKey<<1 mod poly - - PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] -%endif - mov rsp, r14 - - pop r15 - pop r14 - pop r13 - pop r12 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_xmms_avx_asm -%endif -exit_precomp: - - ret - -%ifdef SAFE_PARAM -error_precomp: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - - jmp exit_precomp -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_init_128_avx_gen2( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *iv, -; const u8 *aad, -; u64 aad_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(init,_),function,) -FN_NAME(init,_): - endbranch64 - push r12 - push r13 -%ifidn __OUTPUT_FORMAT__, win64 - push r14 - push r15 - mov r14, rsp - ; xmm6 needs to be maintained for Windows - sub rsp, 1*16 - vmovdqu [rsp + 0*16], xmm6 -%endif - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_init - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_init - - ;; Check IV != NULL - cmp arg3, 0 - jz error_init - - ;; Check if aad_len == 0 - cmp arg5, 0 - jz skip_aad_check_init - - ;; Check aad != NULL (aad_len != 0) - cmp arg4, 0 - jz error_init - -skip_aad_check_init: -%endif - GCM_INIT arg1, arg2, arg3, arg4, arg5 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_xmms_avx_asm -%endif -exit_init: - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6 , [rsp + 0*16] - mov rsp, r14 - pop r15 - pop r14 -%endif - pop r13 - pop r12 - ret - -%ifdef SAFE_PARAM -error_init: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_IV - - ;; Check if aad_len == 0 - cmp arg5, 0 - jz skip_aad_check_error_init - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_init: - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_init -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_init_var_iv_128_avx_gen2 / aes_gcm_init_var_iv_192_avx_gen2 / -; aes_gcm_init_var_iv_256_avx_gen2 ( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(init_var_iv,_),function,) -FN_NAME(init_var_iv,_): - endbranch64 - push r12 - push r13 -%ifidn __OUTPUT_FORMAT__, win64 - push r14 - push r15 - mov r14, rsp - ; xmm6 need to be maintained for Windows - sub rsp, 1*16 - vmovdqu [rsp + 0*16], xmm6 -%endif - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_init_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_init_IV - - ;; Check IV != NULL - cmp arg3, 0 - jz error_init_IV - - ;; Check iv_len != 0 - cmp arg4, 0 - jz error_init_IV - - ;; Check if aad_len == 0 - cmp arg6, 0 - jz skip_aad_check_init_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg5, 0 - jz error_init_IV - -skip_aad_check_init_IV: -%endif - cmp arg4, 12 - je iv_len_12_init_IV - - GCM_INIT arg1, arg2, arg3, arg5, arg6, arg4 - jmp skip_iv_len_12_init_IV - -iv_len_12_init_IV: - GCM_INIT arg1, arg2, arg3, arg5, arg6 - -skip_iv_len_12_init_IV: -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_xmms_avx_asm -%endif -exit_init_IV: - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6 , [rsp + 0*16] - mov rsp, r14 - pop r15 - pop r14 -%endif - pop r13 - pop r12 - ret - -%ifdef SAFE_PARAM -error_init_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_IV - - ;; Check iv_len != 0 - IMB_ERR_CHECK_ZERO arg4, rax, IMB_ERR_IV_LEN - - ;; Check if aad_len == 0 - cmp arg6, 0 - jz skip_aad_check_error_init_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg5, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_init_IV: - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_init_IV -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_update_avx_gen2( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc,_update_),function,) -FN_NAME(enc,_update_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_update_enc - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_update_enc - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz error_update_enc - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_update_enc - - ;; Check out != NULL (plaintext_len != 0) - cmp arg3, 0 - jz error_update_enc - - ;; Check in != NULL (plaintext_len != 0) - cmp arg4, 0 - jz error_update_enc -%endif - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC - -exit_update_enc: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_update_enc: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_update_enc - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (plaintext_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_update_enc: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_update_enc -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_update_avx_gen2( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec,_update_),function,) -FN_NAME(dec,_update_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_update_dec - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_update_dec - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz error_update_dec - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_update_dec - - ;; Check out != NULL (plaintext_len != 0) - cmp arg3, 0 - jz error_update_dec - - ;; Check in != NULL (plaintext_len != 0) - cmp arg4, 0 - jz error_update_dec -%endif - - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC - -exit_update_dec: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_update_dec: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check if plaintext_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_update_dec - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (plaintext_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_update_dec: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_update_dec -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_finalize_avx_gen2( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc,_finalize_),function,) -FN_NAME(enc,_finalize_): - endbranch64 -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc_fin - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc_fin - - ;; Check auth_tag != NULL - cmp arg3, 0 - jz error_enc_fin - - ;; Check auth_tag_len == 0 or > 16 - cmp arg4, 0 - jz error_enc_fin - - cmp arg4, 16 - ja error_enc_fin -%endif - push r12 - -%ifidn __OUTPUT_FORMAT__, win64 - ; xmm6:xmm15 need to be maintained for Windows - sub rsp, 7*16 - vmovdqu [rsp + 0*16], xmm6 - vmovdqu [rsp + 1*16], xmm9 - vmovdqu [rsp + 2*16], xmm10 - vmovdqu [rsp + 3*16], xmm11 - vmovdqu [rsp + 4*16], xmm13 - vmovdqu [rsp + 5*16], xmm14 - vmovdqu [rsp + 6*16], xmm15 -%endif - GCM_COMPLETE arg1, arg2, arg3, arg4 - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm15, [rsp + 6*16] - vmovdqu xmm14, [rsp + 5*16] - vmovdqu xmm13, [rsp + 4*16] - vmovdqu xmm11, [rsp + 3*16] - vmovdqu xmm10, [rsp + 2*16] - vmovdqu xmm9, [rsp + 1*16] - vmovdqu xmm6, [rsp + 0*16] - add rsp, 7*16 -%endif - - pop r12 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_xmms_avx_asm -%endif -exit_enc_fin: - ret - -%ifdef SAFE_PARAM -error_enc_fin: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg4, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg4, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc_fin -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_finalize_avx_gen2( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec,_finalize_),function,) -FN_NAME(dec,_finalize_): - endbranch64 -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec_fin - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec_fin - - ;; Check auth_tag != NULL - cmp arg3, 0 - jz error_dec_fin - - ;; Check auth_tag_len == 0 or > 16 - cmp arg4, 0 - jz error_dec_fin - - cmp arg4, 16 - ja error_dec_fin -%endif - - push r12 - -%ifidn __OUTPUT_FORMAT__, win64 - ; xmm6:xmm15 need to be maintained for Windows - sub rsp, 5*16 - vmovdqu [rsp + 0*16],xmm6 - vmovdqu [rsp + 1*16],xmm9 - vmovdqu [rsp + 2*16],xmm11 - vmovdqu [rsp + 3*16],xmm14 - vmovdqu [rsp + 4*16],xmm15 -%endif - GCM_COMPLETE arg1, arg2, arg3, arg4 - -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm15 , [rsp + 4*16] - vmovdqu xmm14 , [rsp + 3*16] - vmovdqu xmm11 , [rsp + 2*16] - vmovdqu xmm9 , [rsp + 1*16] - vmovdqu xmm6 , [rsp + 0*16] - add rsp, 5*16 -%endif - - pop r12 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_xmms_avx_asm -%endif -exit_dec_fin: - ret - -%ifdef SAFE_PARAM -error_dec_fin: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg4, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg4, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec_fin -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_avx_gen2( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u8 *aad, -; u64 aad_len, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc,_),function,) -FN_NAME(enc,_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc - - ;; Check IV != NULL - cmp arg6, 0 - jz error_enc - - ;; Check auth_tag != NULL - cmp arg9, 0 - jz error_enc - - ;; Check auth_tag_len == 0 or > 16 - cmp arg10, 0 - jz error_enc - - cmp arg10, 16 - ja error_enc - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_enc - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_enc - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_enc - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_enc - -skip_in_out_check_enc: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_enc - - ;; Check aad != NULL (aad_len != 0) - cmp arg7, 0 - jz error_enc - -skip_aad_check_enc: -%endif - GCM_INIT arg1, arg2, arg6, arg7, arg8 - - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC - - GCM_COMPLETE arg1, arg2, arg9, arg10 - -exit_enc: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_enc: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg9, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg10, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg10, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_enc - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_enc: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_error_enc - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg7, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_enc: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_avx_gen2( -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u8 *aad, -; u64 aad_len, -; u8 *auth_tag, -; u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec,_),function,) -FN_NAME(dec,_): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec - - ;; Check IV != NULL - cmp arg6, 0 - jz error_dec - - ;; Check auth_tag != NULL - cmp arg9, 0 - jz error_dec - - ;; Check auth_tag_len == 0 or > 16 - cmp arg10, 0 - jz error_dec - - cmp arg10, 16 - ja error_dec - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_dec - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_dec - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_dec - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_dec - -skip_in_out_check_dec: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_dec - - ;; Check aad != NULL (aad_len != 0) - cmp arg7, 0 - jz error_dec - -skip_aad_check_dec: -%endif - - GCM_INIT arg1, arg2, arg6, arg7, arg8 - - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC - - GCM_COMPLETE arg1, arg2, arg9, arg10 - -exit_dec: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_dec: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg9, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg10, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg10, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_dec - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_dec: - ;; Check if aad_len == 0 - cmp arg8, 0 - jz skip_aad_check_error_dec - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg7, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_dec: - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_var_iv_128_avx_gen2 / aes_gcm_enc_var_iv_192_avx_gen2 / -; aes_gcm_enc_var_iv_256_avx_gen2 -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc_var_iv,_),function,) -FN_NAME(enc_var_iv,_): - - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_enc_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_enc_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_enc_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_enc_IV - - cmp arg11, 16 - ja error_enc_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_enc_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_enc_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_enc_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_enc_IV - -skip_in_out_check_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_enc_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_enc_IV - -skip_aad_check_enc_IV: -%endif - cmp arg7, 12 - je iv_len_12_enc_IV - - GCM_INIT arg1, arg2, arg6, arg8, arg9, arg7 - jmp skip_iv_len_12_enc_IV - -iv_len_12_enc_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9 - -skip_iv_len_12_enc_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC - - GCM_COMPLETE arg1, arg2, arg10, arg11 - -exit_enc_IV: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_enc_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_enc_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_enc_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_enc_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc_IV -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_var_iv_128_avx_gen2 / aes_gcm_dec_var_iv_192_avx_gen2 / -; aes_gcm_dec_var_iv_256_avx_gen2 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec_var_iv,_),function,) -FN_NAME(dec_var_iv,_): - - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_dec_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_dec_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_dec_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_dec_IV - - cmp arg11, 16 - ja error_dec_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_dec_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_dec_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_dec_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_dec_IV - -skip_in_out_check_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_dec_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_dec_IV - -skip_aad_check_dec_IV: -%endif - cmp arg7, 12 - je iv_len_12_dec_IV - - GCM_INIT arg1, arg2, arg6, arg8, arg9, arg7 - jmp skip_iv_len_12_dec_IV - -iv_len_12_dec_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9 - -skip_iv_len_12_dec_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC - - GCM_COMPLETE arg1, arg2, arg10, arg11 - -exit_dec_IV: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_dec_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_dec_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_dec_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_dec_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec_IV -%endif - -%ifdef GCM128_MODE -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void ghash_pre_avx_gen2 -; (const void *key, struct gcm_key_data *key_data) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(ghash_pre_avx_gen2,function,) -ghash_pre_avx_gen2: - endbranch64 -;; Parameter is passed through register -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key != NULL - cmp arg1, 0 - jz error_ghash_pre - - ;; Check key_data != NULL - cmp arg2, 0 - jz error_ghash_pre -%endif - -%ifidn __OUTPUT_FORMAT__, win64 - push r14 - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 ; align rsp to 64 bytes - - ; only xmm6 needs to be maintained - vmovdqa [rsp + LOCAL_STORAGE + 0*16],xmm6 -%endif - vmovdqu xmm6, [arg1] - vpshufb xmm6, [rel SHUF_MASK] - ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; - vmovdqa xmm2, xmm6 - vpsllq xmm6, xmm6, 1 - vpsrlq xmm2, xmm2, 63 - vmovdqa xmm1, xmm2 - vpslldq xmm2, xmm2, 8 - vpsrldq xmm1, xmm1, 8 - vpor xmm6, xmm6, xmm2 - ;reduction - vpshufd xmm2, xmm1, 00100100b - vpcmpeqd xmm2, [rel TWOONE] - vpand xmm2, xmm2, [rel POLY] - vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vmovdqu [arg2 + HashKey], xmm6 ; store HashKey<<1 mod poly - - PRECOMPUTE arg2, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 - -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_xmms_avx_asm -%endif -%ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] - mov rsp, r14 - pop r14 - -%endif -exit_ghash_pre: - ret - -%ifdef SAFE_PARAM -error_ghash_pre: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_KEY - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_EXP_KEY - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_ghash_pre -%endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void ghash_avx_gen2 -; const struct gcm_key_data *key_data, -; const void *in, -; const u64 in_len, -; void *io_tag, -; const u64 tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(ghash_avx_gen2,function,) -ghash_avx_gen2: - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_ghash - - ;; Check in != NULL - cmp arg2, 0 - jz error_ghash - - ;; Check in_len != 0 - cmp arg3, 0 - jz error_ghash - - ;; Check tag != NULL - cmp arg4, 0 - jz error_ghash - - ;; Check tag_len != 0 - cmp arg5, 0 - jz error_ghash -%endif - ;; copy tag to xmm0 - vmovdqu xmm0, [arg4] - vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap - - CALC_AAD_HASH arg2, arg3, xmm0, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ - r10, r11, r12, r13, rax - - vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap - - simd_store_avx arg4, xmm0, arg5, r12, rax - -exit_ghash: - FUNC_RESTORE - ret - -%ifdef SAFE_PARAM -error_ghash: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check in != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_SRC - - ;; Check in_len != 0 - IMB_ERR_CHECK_ZERO arg3, rax, IMB_ERR_AUTH_LEN - - ;; Check tag != NULL - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_AUTH - - ;; Check tag_len != 0 - IMB_ERR_CHECK_ZERO arg5, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - - jmp exit_ghash -%endif - -%endif ;; GCM128_MODE - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; PARTIAL_BLOCK_GMAC: Handles the tag partial blocks between update calls. -; Requires the input data be at least 1 byte long. -; Input: gcm_key_data (GDATA_KEY), gcm_context_data (GDATA_CTX), input text (PLAIN_IN), -; input text length (PLAIN_LEN), hash subkey (HASH_SUBKEY). -; Output: Updated GDATA_CTX -; Clobbers rax, r10, r12, r13, r15, xmm0, xmm1, xmm2, xmm3, xmm5, xmm6, xmm9, xmm10, xmm11 -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro PARTIAL_BLOCK_GMAC 7 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%PLAIN_IN %3 -%define %%PLAIN_LEN %4 -%define %%DATA_OFFSET %5 -%define %%AAD_HASH %6 -%define %%HASH_SUBKEY %7 - - mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 - ; Leave Macro if no partial blocks - je %%_partial_block_done - - ; Read in input data without over reading - cmp %%PLAIN_LEN, 16 - jl %%_fewer_than_16_bytes - ; If more than 16 bytes of data, just fill the xmm register - VXLDR xmm1, [%%PLAIN_IN] - jmp %%_data_read - -%%_fewer_than_16_bytes: - lea r10, [%%PLAIN_IN] - READ_SMALL_DATA_INPUT_AVX xmm1, r10, %%PLAIN_LEN, rax, r12, r15 - - ; Finished reading in data -%%_data_read: - - lea r12, [rel SHIFT_MASK] - ; Adjust the shuffle mask pointer to be able to shift r13 bytes - ; (16-r13 is the number of bytes in plaintext mod 16) - add r12, r13 - ; Get the appropriate shuffle mask - vmovdqu xmm2, [r12] - vmovdqa xmm3, xmm1 - - mov r15, %%PLAIN_LEN - add r15, r13 - ; Set r15 to be the amount of data left in PLAIN_IN after filling the block - sub r15, 16 - ; Determine if partial block is not being filled and shift mask accordingly - jge %%_no_extra_mask_1 - sub r12, r15 -%%_no_extra_mask_1: - - ; Get the appropriate mask to mask out bottom r13 bytes of xmm3 - vmovdqu xmm1, [r12 + ALL_F-SHIFT_MASK] - - vpand xmm3, xmm1 - vpshufb xmm3, [rel SHUF_MASK] - vpshufb xmm3, xmm2 - vpxor %%AAD_HASH, xmm3 - - cmp r15,0 - jl %%_partial_incomplete_1 - - ; GHASH computation for the last <16 Byte block - GHASH_MUL %%AAD_HASH, %%HASH_SUBKEY, xmm0, xmm10, xmm11, xmm5, xmm6 - xor rax, rax - mov [%%GDATA_CTX + PBlockLen], rax - jmp %%_ghash_done -%%_partial_incomplete_1: -%ifidn __OUTPUT_FORMAT__, win64 - mov rax, %%PLAIN_LEN - add [%%GDATA_CTX + PBlockLen], rax -%else - add [%%GDATA_CTX + PBlockLen], %%PLAIN_LEN -%endif -%%_ghash_done: - vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH - - cmp r15, 0 - jl %%_partial_fill - - mov r12, 16 - ; Set r12 to be the number of bytes to skip after this macro - sub r12, r13 - - jmp %%offset_set -%%_partial_fill: - mov r12, %%PLAIN_LEN -%%offset_set: - mov %%DATA_OFFSET, r12 -%%_partial_block_done: -%endmacro ; PARTIAL_BLOCK_GMAC - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void imb_aes_gmac_update_128_avx_gen2 / imb_aes_gmac_update_192_avx_gen2 / -; imb_aes_gmac_update_256_avx_gen2 -; const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; const u8 *in, -; const u64 msg_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(GMAC_FN_NAME(update),function,) -GMAC_FN_NAME(update): - endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET -%endif - ;; Check if msg_len == 0 - cmp arg4, 0 - je exit_gmac_update - -%ifdef SAFE_PARAM - ;; Check key_data != NULL - cmp arg1, 0 - jz error_gmac_update - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_gmac_update - - ;; Check in != NULL (msg_len != 0) - cmp arg3, 0 - jz error_gmac_update -%endif - - ; Increment size of "AAD length" for GMAC - add [arg2 + AadLen], arg4 - - ;; Deal with previous partial block - xor r11, r11 - vmovdqu xmm13, [arg1 + HashKey] - vmovdqu xmm8, [arg2 + AadHash] - - PARTIAL_BLOCK_GMAC arg1, arg2, arg3, arg4, r11, xmm8, xmm13 - - ; CALC_AAD_HASH needs to deal with multiple of 16 bytes - sub arg4, r11 - add arg3, r11 - - vmovq xmm7, arg4 ; Save remaining length - and arg4, -16 ; Get multiple of 16 bytes - - or arg4, arg4 - jz no_full_blocks - - ;; Calculate GHASH of this segment - CALC_AAD_HASH arg3, arg4, xmm8, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ - r10, r11, r12, r13, rax - vmovdqu [arg2 + AadHash], xmm8 ; ctx_data.aad hash = aad_hash - -no_full_blocks: - add arg3, arg4 ; Point at partial block - - vmovq arg4, xmm7 ; Restore original remaining length - and arg4, 15 - jz exit_gmac_update - - ; Save next partial block - mov [arg2 + PBlockLen], arg4 - READ_SMALL_DATA_INPUT_AVX xmm1, arg3, arg4, r11, r12, r13 - vpshufb xmm1, [rel SHUF_MASK] - vpxor xmm8, xmm1 - vmovdqu [arg2 + AadHash], xmm8 - -exit_gmac_update: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_gmac_update: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_SRC - - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_gmac_update -%endif - -mksection stack-noexec diff --git a/lib/avx_t1/kasumi_avx.c b/lib/avx_t1/kasumi_avx.c index 697e4953d8de30495f002b486f9df1d3068d1e93..eb2f6bac63ba8df1c04896f442f27d33bbb8dd9e 100644 --- a/lib/avx_t1/kasumi_avx.c +++ b/lib/avx_t1/kasumi_avx.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm index c0a68a4f759b625b44e567ec64d21313410e9ce7..f902fa4104f70c0ce8ebb6cebb6ef3d9d651ef02 100644 --- a/lib/avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %ifndef AES_CBC_ENC_X8 %define AES_CBC_ENC_X8 aes_cbc_enc_128_x8 diff --git a/lib/avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm index e44dadebfa4e65ec3a3b165a3d33cf6a12f44027..a6bac0a8c478759ae118f2ec09d494ef7d19c4c3 100644 --- a/lib/avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" %ifndef AES_CBC_ENC_X8 diff --git a/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_flush_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_flush_avx.asm index 85148570caf68bcfe6a5f5c13f41ca8101330159..530d4032c590cb268239c81e793a205a26b2d214 100644 --- a/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %define NUM_LANES 8 diff --git a/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_submit_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_submit_avx.asm index a18c5dd34a300dcc474f8eb37975fb306ca5d096..aabfd6b5f7b4da846956a2a6cadcee3168bca608 100644 --- a/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" %define NUM_LANES 8 diff --git a/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm index fc82044ebc10fe9d55a685c196442b38ace51585..2f43a83fc32d2a4fe0ae0ac5120e11a1ad832e43 100644 --- a/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/memcpy.asm" +%include "include/memcpy.inc" %ifndef AES_CBC_MAC diff --git a/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm index f3a8bdb49172994af271ba23f3fa004fe6e7b4a2..6a730617538fbfac157f486d2f805f400e55a603 100644 --- a/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,14 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" %ifndef AES_CBC_MAC %define AES_CBC_MAC aes128_cbc_mac_x8 diff --git a/lib/avx_t1/mb_mgr_aes128_xcbc_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_xcbc_flush_x8_avx.asm index 57f10267fd67db85cec7cbd25ba1b40bc0187075..76b1d2b6ef7d48d0c392858dae59b5e0cae457ef 100644 --- a/lib/avx_t1/mb_mgr_aes128_xcbc_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_xcbc_flush_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %ifndef AES_XCBC_X8 %define AES_XCBC_X8 aes_xcbc_mac_128_x8 diff --git a/lib/avx_t1/mb_mgr_aes128_xcbc_submit_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_xcbc_submit_x8_avx.asm index 525d4d24926631e468073c2e9f85c6cc53164562..77578930881f5918b3725f8db7e0a21125f6236f 100644 --- a/lib/avx_t1/mb_mgr_aes128_xcbc_submit_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_xcbc_submit_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" -%include "include/memcpy.asm" +%include "include/memcpy.inc" %include "include/const.inc" %ifndef AES_XCBC_X8 diff --git a/lib/avx_t1/mb_mgr_aes192_cbc_enc_flush_avx.asm b/lib/avx_t1/mb_mgr_aes192_cbc_enc_flush_avx.asm index 13797d301382fb58e6cd1a928495e352de7fe644..f85a821a1c81d49f2b2785c745951f13b4bb36c4 100644 --- a/lib/avx_t1/mb_mgr_aes192_cbc_enc_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_aes192_cbc_enc_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_aes192_cbc_enc_submit_avx.asm b/lib/avx_t1/mb_mgr_aes192_cbc_enc_submit_avx.asm index 1dd9ce6cf74e1e082c89617abd53628b1dcb23fb..7dace4b7743daad1005d44a498414b0b331b2a91 100644 --- a/lib/avx_t1/mb_mgr_aes192_cbc_enc_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_aes192_cbc_enc_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_aes256_cbc_enc_flush_avx.asm b/lib/avx_t1/mb_mgr_aes256_cbc_enc_flush_avx.asm index 75e8504214f2f4a38268c109f1f46d0b58ac8064..623b59932c1d8c38e2cac7abe82990d009780d42 100644 --- a/lib/avx_t1/mb_mgr_aes256_cbc_enc_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_cbc_enc_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_aes256_cbc_enc_submit_avx.asm b/lib/avx_t1/mb_mgr_aes256_cbc_enc_submit_avx.asm index 8aa20d9dd6adbf812125923366f3e9325c288222..14f13f3355bddfe2eaf2b7771544e6cd2dc62006 100644 --- a/lib/avx_t1/mb_mgr_aes256_cbc_enc_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_cbc_enc_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm index 186f319f5e745a6d41096d951305ea728a3545fd..b3e6cdda335aa3fadb1f15a4395cae1d9c08ccb2 100644 --- a/lib/avx_t1/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm index 62aa60ef4297193bcd9a21fe12cc3dde4f2eb0b7..4884e2b3b4942930d9de1b083672426620c480b3 100644 --- a/lib/avx_t1/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_avx.c b/lib/avx_t1/mb_mgr_avx.c index 94ed6caaada7a7508e57cb39331326e4d0e2906c..e0a8fe022bd26a4e855c9979afbf485354405702 100644 --- a/lib/avx_t1/mb_mgr_avx.c +++ b/lib/avx_t1/mb_mgr_avx.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_avx_t1.c b/lib/avx_t1/mb_mgr_avx_t1.c index 2ed82c0fccf3d7a86cffbc7ba2da9081e0129d77..05de2dc5d5865065dda591a81d9d4fadf8b22af5 100644 --- a/lib/avx_t1/mb_mgr_avx_t1.c +++ b/lib/avx_t1/mb_mgr_avx_t1.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -44,7 +44,7 @@ #include "include/cpu_feature.h" #include "include/aesni_emu.h" #include "include/error.h" -#include "include/arch_sse_type1.h" /* snow3g */ +#include "include/arch_sse_type1.h" /* snow3g, gcm */ #include "include/arch_avx_type1.h" #include "include/ooo_mgr_reset.h" @@ -66,6 +66,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx_t1 #define SUBMIT_HASH_BURST submit_hash_burst_avx_t1 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx_t1 +#define SET_SUITE_ID_FN set_suite_id_avx_t1 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX @@ -77,12 +78,12 @@ #define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX /* AES-GCM */ -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen2 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen2 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen2 +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_sse +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx @@ -358,6 +359,7 @@ init_mb_mgr_avx_t1_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_avx; state->keyexp_192 = aes_keyexp_192_avx; @@ -382,6 +384,7 @@ init_mb_mgr_avx_t1_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_avx; state->aes128_cfb_one = aes_cfb_128_one_avx; + state->aes256_cfb_one = aes_cfb_256_one_avx; state->eea3_1_buffer = zuc_eea3_1_buffer_avx; state->eea3_4_buffer = zuc_eea3_4_buffer_avx; @@ -434,49 +437,52 @@ init_mb_mgr_avx_t1_internal(IMB_MGR *state, const int reset_mgrs) state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx; state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; - state->gcm128_enc = aes_gcm_enc_128_avx_gen2; - state->gcm192_enc = aes_gcm_enc_192_avx_gen2; - state->gcm256_enc = aes_gcm_enc_256_avx_gen2; - state->gcm128_dec = aes_gcm_dec_128_avx_gen2; - state->gcm192_dec = aes_gcm_dec_192_avx_gen2; - state->gcm256_dec = aes_gcm_dec_256_avx_gen2; - state->gcm128_init = aes_gcm_init_128_avx_gen2; - state->gcm192_init = aes_gcm_init_192_avx_gen2; - state->gcm256_init = aes_gcm_init_256_avx_gen2; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen2; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen2; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen2; - state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen2; - state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen2; - state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen2; - state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen2; - state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen2; - state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen2; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen2; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen2; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen2; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen2; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen2; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen2; - state->gcm128_precomp = aes_gcm_precomp_128_avx_gen2; - state->gcm192_precomp = aes_gcm_precomp_192_avx_gen2; - state->gcm256_precomp = aes_gcm_precomp_256_avx_gen2; - state->gcm128_pre = aes_gcm_pre_128_avx_gen2; - state->gcm192_pre = aes_gcm_pre_192_avx_gen2; - state->gcm256_pre = aes_gcm_pre_256_avx_gen2; - - state->ghash = ghash_avx_gen2; - state->ghash_pre = ghash_pre_avx_gen2; - - state->gmac128_init = imb_aes_gmac_init_128_avx_gen2; - state->gmac192_init = imb_aes_gmac_init_192_avx_gen2; - state->gmac256_init = imb_aes_gmac_init_256_avx_gen2; - state->gmac128_update = imb_aes_gmac_update_128_avx_gen2; - state->gmac192_update = imb_aes_gmac_update_192_avx_gen2; - state->gmac256_update = imb_aes_gmac_update_256_avx_gen2; - state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen2; - state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen2; - state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen2; + state->gcm128_enc = aes_gcm_enc_128_sse; + state->gcm192_enc = aes_gcm_enc_192_sse; + state->gcm256_enc = aes_gcm_enc_256_sse; + state->gcm128_dec = aes_gcm_dec_128_sse; + state->gcm192_dec = aes_gcm_dec_192_sse; + state->gcm256_dec = aes_gcm_dec_256_sse; + state->gcm128_init = aes_gcm_init_128_sse; + state->gcm192_init = aes_gcm_init_192_sse; + state->gcm256_init = aes_gcm_init_256_sse; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_sse; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_sse; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_sse; + state->gcm128_enc_update = aes_gcm_enc_128_update_sse; + state->gcm192_enc_update = aes_gcm_enc_192_update_sse; + state->gcm256_enc_update = aes_gcm_enc_256_update_sse; + state->gcm128_dec_update = aes_gcm_dec_128_update_sse; + state->gcm192_dec_update = aes_gcm_dec_192_update_sse; + state->gcm256_dec_update = aes_gcm_dec_256_update_sse; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse; + state->gcm128_precomp = aes_gcm_precomp_128_sse; + state->gcm192_precomp = aes_gcm_precomp_192_sse; + state->gcm256_precomp = aes_gcm_precomp_256_sse; + state->gcm128_pre = aes_gcm_pre_128_sse; + state->gcm192_pre = aes_gcm_pre_192_sse; + state->gcm256_pre = aes_gcm_pre_256_sse; + + state->ghash = ghash_sse; + state->ghash_pre = ghash_pre_sse; + + state->gmac128_init = imb_aes_gmac_init_128_sse; + state->gmac192_init = imb_aes_gmac_init_192_sse; + state->gmac256_init = imb_aes_gmac_init_256_sse; + state->gmac128_update = imb_aes_gmac_update_128_sse; + state->gmac192_update = imb_aes_gmac_update_192_sse; + state->gmac256_update = imb_aes_gmac_update_256_sse; + state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; + state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; + state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_avx; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_avx; } #include "mb_mgr_code.h" diff --git a/lib/avx_t1/mb_mgr_hmac_md5_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_md5_flush_avx.asm index f8ab51e4e372079792201eeadc0b4989b1231240..17da322ca0bf2240cce3c001cf49ee1543923a7e 100644 --- a/lib/avx_t1/mb_mgr_hmac_md5_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_md5_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern md5_x4x2_avx diff --git a/lib/avx_t1/mb_mgr_hmac_md5_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_md5_submit_avx.asm index e74f729ba313cb05be66e1dd099c8206fc5107f5..edf3d2163a472b49cfce9b27af65f82de851a764 100644 --- a/lib/avx_t1/mb_mgr_hmac_md5_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_md5_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/memcpy.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/memcpy.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" extern md5_x4x2_avx diff --git a/lib/avx_t1/mb_mgr_hmac_sha1_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha1_flush_avx.asm index 01611f54ba44059501d1433c84add67ddccc554a..e54e3e08164d086eb1c8d2e7a2cea136544b450e 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha1_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha1_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern sha1_mult_avx diff --git a/lib/avx_t1/mb_mgr_hmac_sha1_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha1_submit_avx.asm index 160e3373a2e8dce0401230620fa99fd591c6b784..54efee0910e18b77490d9de2f96709f2cdbbcf09 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha1_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha1_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" extern sha1_mult_avx diff --git a/lib/avx_t1/mb_mgr_hmac_sha224_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha224_flush_avx.asm index 1efdcea168061a0c97b8f3b402964ea77b1c1741..af3bbe52380c9fe914365f00caebb1906cc6a499 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha224_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha224_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_hmac_sha224_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha224_submit_avx.asm index d26a0ab641c1d0bd908f0e9303dee47355f72585..2000ba97065efeb26d2fcfda25472990f74d948f 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha224_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha224_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_hmac_sha256_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha256_flush_avx.asm index 7ce1825fe6362a807f8e6b42d5198a0b66849cc3..fe6c8f02f265567d4400ec2f85283a99e418e36f 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha256_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha256_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern sha_256_mult_avx diff --git a/lib/avx_t1/mb_mgr_hmac_sha256_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha256_submit_avx.asm index 09894723be864975833126263a458a33e2353397..f95ae24bb40d86db4c85fc89b286729e6b41d82f 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha256_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha256_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" extern sha_256_mult_avx diff --git a/lib/avx_t1/mb_mgr_hmac_sha384_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha384_flush_avx.asm index 9ee3c12431efded713bea3d4f4180f1de0408e2e..6505c88e4909bf2e8fc258f3d241753d35f31d59 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha384_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha384_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_hmac_sha384_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha384_submit_avx.asm index 70a90ba8290d8226793037e65f571f49c298247b..d58fb927f0c952e0d4ed37e5afaa70dc5f2119e7 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha384_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha384_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/mb_mgr_hmac_sha512_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha512_flush_avx.asm index e7958a81736a0ca91933bc644e24f171524ec8bc..e8d4292d25691554de3a54422e8470232c494762 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha512_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha512_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern sha512_x2_avx diff --git a/lib/avx_t1/mb_mgr_hmac_sha512_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha512_submit_avx.asm index bd2d76d0a32574dbb70a75763bc32b531bd345fc..ff482c29cb2972ac431b376327b91e7c2f274ca9 100644 --- a/lib/avx_t1/mb_mgr_hmac_sha512_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha512_submit_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" extern sha512_x2_avx diff --git a/lib/avx_t1/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx_t1/mb_mgr_zuc_submit_flush_avx.asm index 92fb7208040bc3a16e6ef0eb44d5a813298c117c..57b4c0519829118f3018d05629160250bc85394d 100644 --- a/lib/avx_t1/mb_mgr_zuc_submit_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_zuc_submit_flush_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,13 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" %include "include/cet.inc" -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_avx %define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_avx diff --git a/lib/avx_t1/md5_x4x2_avx.asm b/lib/avx_t1/md5_x4x2_avx.asm index 32528d188c12de630280c62a4fce2d14235130b6..ce692928e8f9c6a12992dc2fa59b149613cb841d 100644 --- a/lib/avx_t1/md5_x4x2_avx.asm +++ b/lib/avx_t1/md5_x4x2_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -36,9 +36,9 @@ ;; ;; clobbers xmm0-15 -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" extern MD5_TABLE diff --git a/lib/avx_t1/memcpy_avx.asm b/lib/avx_t1/memcpy_avx.asm index a92bbb915c8aea535e6decb781c400d96cd5c052..501f8850b0d5f2a047d38b7136e77940fbc21f6b 100644 --- a/lib/avx_t1/memcpy_avx.asm +++ b/lib/avx_t1/memcpy_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,8 +25,8 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/memcpy.inc" %ifdef LINUX %define arg1 rdi diff --git a/lib/avx_t1/pon_by8_avx.asm b/lib/avx_t1/pon_by8_avx.asm index 74f2f27d80a3f4c60b6226883aa5f353bf435e90..058d1533d3266b138564324724fbfe7eb8f14526 100644 --- a/lib/avx_t1/pon_by8_avx.asm +++ b/lib/avx_t1/pon_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,10 +27,10 @@ %use smartalign -%include "include/imb_job.asm" -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" +%include "include/imb_job.inc" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/avx_t1/sha1_one_block_avx.asm b/lib/avx_t1/sha1_one_block_avx.asm index af6b556581e0d9e3bcb9e844b583ce00234766f2..b1a609cbc488fe4ca0d0fd62fea980aba117c419 100644 --- a/lib/avx_t1/sha1_one_block_avx.asm +++ b/lib/avx_t1/sha1_one_block_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,8 +27,8 @@ ; SHA1 code, hybrid, rolled, interleaved ; Uses AVX instructions -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/avx_t1/sha1_x4_avx.asm b/lib/avx_t1/sha1_x4_avx.asm index b67ef36019681aee17e85fc90a7af7648f3aef49..948f71ceda09a968ac17cf32aa24ed3359b5d750 100644 --- a/lib/avx_t1/sha1_x4_avx.asm +++ b/lib/avx_t1/sha1_x4_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/avx_t1/sha224_one_block_avx.asm b/lib/avx_t1/sha224_one_block_avx.asm index f80b557989328cdf97b024dbd58b0f3a471eb422..7c69e36a84e3759a617f4ebe89f08a0f5de8615d 100644 --- a/lib/avx_t1/sha224_one_block_avx.asm +++ b/lib/avx_t1/sha224_one_block_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/sha256_mult_avx.asm b/lib/avx_t1/sha256_mult_avx.asm index 598d4cdae518946fd6668e7fc6bfe461b8d11542..304912fe8f3dc2352698420d84e72d998287f6ca 100644 --- a/lib/avx_t1/sha256_mult_avx.asm +++ b/lib/avx_t1/sha256_mult_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -38,9 +38,9 @@ ;; ;; clobbers xmm0-15 -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" extern K256_4 diff --git a/lib/avx_t1/sha256_one_block_avx.asm b/lib/avx_t1/sha256_one_block_avx.asm index c50cd0a4b1777f4ea16c93c5789d837048e1165d..cd99354f10bc07990e8bb8a6362904d47884cfdc 100644 --- a/lib/avx_t1/sha256_one_block_avx.asm +++ b/lib/avx_t1/sha256_one_block_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,8 +27,8 @@ ; This code schedules 1 blocks at a time, with 4 lanes per block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/avx_t1/sha384_one_block_avx.asm b/lib/avx_t1/sha384_one_block_avx.asm index fc9bf7359aeba31a486d764cdb55570b43e7f9d6..5be46bbd696060a332ac9b89480b7c206d8284b5 100644 --- a/lib/avx_t1/sha384_one_block_avx.asm +++ b/lib/avx_t1/sha384_one_block_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/sha512_one_block_avx.asm b/lib/avx_t1/sha512_one_block_avx.asm index 36681a62cc335b89f24d96bd0e6f20cff43baf4f..592ebc59001b212ab48a8b8b2730deafb43f654e 100644 --- a/lib/avx_t1/sha512_one_block_avx.asm +++ b/lib/avx_t1/sha512_one_block_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,8 +27,8 @@ ; This code schedules 1 blocks at a time, with 4 lanes per block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %define VMOVDQ vmovdqu ;; assume buffers not aligned diff --git a/lib/avx_t1/sha512_x2_avx.asm b/lib/avx_t1/sha512_x2_avx.asm index 865008db39b018a5133018151ef4fb032da7a329..d5b2e392be4c8e9be28c9298b22fc491017aaeee 100644 --- a/lib/avx_t1/sha512_x2_avx.asm +++ b/lib/avx_t1/sha512_x2_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -39,9 +39,9 @@ ;; ;; clobbers xmm0-15 -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" extern K512_2 mksection .rodata diff --git a/lib/avx_t1/sha_avx.c b/lib/avx_t1/sha_avx.c index 23cbd17a5967e9744238dcc34981ae07f11e207e..b0c7845e88daf374f4d58884506fee58c15aad0d 100644 --- a/lib/avx_t1/sha_avx.c +++ b/lib/avx_t1/sha_avx.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -26,26 +26,16 @@ *******************************************************************************/ #include "include/sha_generic.h" +#include "include/arch_avx_type1.h" /* ========================================================================== */ /* One block SHA1 computation for IPAD / OPAD usage only */ void sha1_one_block_avx(const void *data, void *digest) { - sha_generic_1block(data, digest, 1 /* AVX */, 1 /* SHA1 */); + sha_generic_1block(data, digest, ARCH_AVX, 1 /* SHA1 */); } -void sha1_one_block_avx2(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 1 /* SHA1 */); -} - -void sha1_one_block_avx512(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 1 /* SHA1 */); -} - - /* ========================================================================== */ /* * SHA1 API for use in HMAC-SHA1 when key is longer than the block size @@ -53,19 +43,7 @@ void sha1_one_block_avx512(const void *data, void *digest) void sha1_avx(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 1 /* AVX */, 1, IMB_SHA1_BLOCK_SIZE, - SHA1_PAD_SIZE); -} - -void sha1_avx2(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 1, IMB_SHA1_BLOCK_SIZE, - SHA1_PAD_SIZE); -} - -void sha1_avx512(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 1, IMB_SHA1_BLOCK_SIZE, + sha_generic(data, length, digest, ARCH_AVX, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE); } @@ -74,17 +52,7 @@ void sha1_avx512(const void *data, const uint64_t length, void *digest) void sha224_one_block_avx(const void *data, void *digest) { - sha_generic_1block(data, digest, 1 /* AVX */, 224 /* SHA224 */); -} - -void sha224_one_block_avx2(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 224 /* SHA224 */); -} - -void sha224_one_block_avx512(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 224 /* SHA224 */); + sha_generic_1block(data, digest, ARCH_AVX, 224 /* SHA224 */); } /* ========================================================================== */ @@ -93,19 +61,7 @@ void sha224_one_block_avx512(const void *data, void *digest) */ void sha224_avx(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 1 /* AVX */, 224, - IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE); -} - -void sha224_avx2(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 224, - IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE); -} - -void sha224_avx512(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 224, + sha_generic(data, length, digest, ARCH_AVX, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE); } @@ -113,17 +69,7 @@ void sha224_avx512(const void *data, const uint64_t length, void *digest) /* One block SHA256 computation for IPAD / OPAD usage only */ void sha256_one_block_avx(const void *data, void *digest) { - sha_generic_1block(data, digest, 1 /* AVX */, 256 /* SHA256 */); -} - -void sha256_one_block_avx2(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 256 /* SHA256 */); -} - -void sha256_one_block_avx512(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 256 /* SHA256 */); + sha_generic_1block(data, digest, ARCH_AVX, 256 /* SHA256 */); } /* ========================================================================== */ @@ -132,19 +78,7 @@ void sha256_one_block_avx512(const void *data, void *digest) */ void sha256_avx(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 1 /* AVX */, 256, - IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE); -} - -void sha256_avx2(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 256, - IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE); -} - -void sha256_avx512(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 256, + sha_generic(data, length, digest, ARCH_AVX, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE); } @@ -152,17 +86,7 @@ void sha256_avx512(const void *data, const uint64_t length, void *digest) /* One block SHA384 computation for IPAD / OPAD usage only */ void sha384_one_block_avx(const void *data, void *digest) { - sha_generic_1block(data, digest, 1 /* AVX */, 384 /* SHA384 */); -} - -void sha384_one_block_avx2(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 384 /* SHA384 */); -} - -void sha384_one_block_avx512(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 384 /* SHA384 */); + sha_generic_1block(data, digest, ARCH_AVX, 384 /* SHA384 */); } /* ========================================================================== */ @@ -171,19 +95,7 @@ void sha384_one_block_avx512(const void *data, void *digest) */ void sha384_avx(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 1 /* AVX */, 384, - IMB_SHA_384_BLOCK_SIZE, SHA384_PAD_SIZE); -} - -void sha384_avx2(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 384, - IMB_SHA_384_BLOCK_SIZE, SHA384_PAD_SIZE); -} - -void sha384_avx512(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 384, + sha_generic(data, length, digest, ARCH_AVX, 384, IMB_SHA_384_BLOCK_SIZE, SHA384_PAD_SIZE); } @@ -191,17 +103,7 @@ void sha384_avx512(const void *data, const uint64_t length, void *digest) /* One block SHA512 computation for IPAD / OPAD usage only */ void sha512_one_block_avx(const void *data, void *digest) { - sha_generic_1block(data, digest, 1 /* AVX */, 512 /* SHA512 */); -} - -void sha512_one_block_avx2(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 512 /* SHA512 */); -} - -void sha512_one_block_avx512(const void *data, void *digest) -{ - sha_generic_1block(data, digest, 1 /* AVX */, 512 /* SHA512 */); + sha_generic_1block(data, digest, ARCH_AVX, 512 /* SHA512 */); } /* ========================================================================== */ @@ -210,18 +112,6 @@ void sha512_one_block_avx512(const void *data, void *digest) */ void sha512_avx(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 1 /* AVX */, 512, - IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE); -} - -void sha512_avx2(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 512, - IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE); -} - -void sha512_avx512(const void *data, const uint64_t length, void *digest) -{ - sha_generic(data, length, digest, 1 /* AVX */, 512, + sha_generic(data, length, digest, ARCH_AVX, 512, IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE); } diff --git a/lib/avx_t1/sha_mb_avx.c b/lib/avx_t1/sha_mb_avx.c index b8026e7da11b0fd27e0075e2d862142cce48858b..b156999dc20b95c3edea0fa1c63617dd60954098 100644 --- a/lib/avx_t1/sha_mb_avx.c +++ b/lib/avx_t1/sha_mb_avx.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/snow3g_avx.c b/lib/avx_t1/snow3g_avx.c index 22a871acd9f651cbf4d452815c246f751426fb12..f1285b06a8e5af0f9d8a18e17e8f3b25de3ddb93 100644 --- a/lib/avx_t1/snow3g_avx.c +++ b/lib/avx_t1/snow3g_avx.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/snow3g_uia2_by4_avx.asm b/lib/avx_t1/snow3g_uia2_by4_avx.asm index b8b56337ddfee81e469eed4b1ef85fb48395f1e6..309b4b70b939a6b89d6267af374998f26e008762 100644 --- a/lib/avx_t1/snow3g_uia2_by4_avx.asm +++ b/lib/avx_t1/snow3g_uia2_by4_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/cet.inc" -%include "include/memcpy.asm" +%include "include/memcpy.inc" %include "include/const.inc" %define APPEND(a,b) a %+ b %define APPEND3(a,b,c) a %+ b %+ c diff --git a/lib/avx_t1/snow_v_avx.asm b/lib/avx_t1/snow_v_avx.asm index 2d06cb7f8e2d2b21620c6258206fd14139bad57a..8286da04f9605b1303bf0d580e32a6f1cec0e2b6 100644 --- a/lib/avx_t1/snow_v_avx.asm +++ b/lib/avx_t1/snow_v_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -30,11 +30,11 @@ ;; abstract : 'A new SNOW stream cipher called SNOW-V' ;; https://eprint.iacr.org/2018/1143.pdf -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" -%include "include/imb_job.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" %ifndef SNOW_V @@ -75,14 +75,20 @@ dq 0x20646b4578656c41 dq 0x6d6f6854676E694a %ifdef LINUX - %define arg1 rdi - %define offset rcx + %define arg1 rdi + %define offset rcx %else - %define arg1 rcx - %define offset r8 + %define arg1 rcx + %define offset r8 %endif -%define job arg1 +%define job arg1 + +;; stack frame for saving registers (windows only) +struc STACK +_xmm_save: resq 10 * 2 ; space for 10 xmm registers +_rsp_save: resq 1 ; space for rsp pointer +endstruc mksection .text @@ -121,13 +127,13 @@ mksection .text %macro SNOW_V_KEYSTREAM 4 ;; all input is expected to be xmm registers -%define %%KEYSTREAM %1 ;; [out] 128 bit keystream -%define %%LFSR_B_HDQ %2 ;; [in] 128 bit LFSR_B_HDQ (b15, ..., b8) -%define %%FSM_R1 %3 ;; [in] 128 bit FSM: R1 -%define %%FSM_R2 %4 ;; [in] 128 bit FSM: R2 +%define %%KEYSTREAM %1 ;; [out] 128 bit keystream +%define %%LFSR_B_HDQ %2 ;; [in] 128 bit LFSR_B_HDQ (b15, ..., b8) +%define %%FSM_R1 %3 ;; [in] 128 bit FSM: R1 +%define %%FSM_R2 %4 ;; [in] 128 bit FSM: R2 - vpaddd %%KEYSTREAM, %%LFSR_B_HDQ, %%FSM_R1 - vpxor %%KEYSTREAM, %%KEYSTREAM, %%FSM_R2 + vpaddd %%KEYSTREAM, %%LFSR_B_HDQ, %%FSM_R1 + vpxor %%KEYSTREAM, %%KEYSTREAM, %%FSM_R2 %endmacro ;; SNOW_V_KEYSTREAM @@ -147,14 +153,14 @@ mksection .text %define %%TEMP1 %5 ;; [clobbered] 128 bit register %define %%TEMP2 %6 ;; [clobbered] 128 bit register - vpxor %%TEMP2, %%LFSR_A_LDQ, %%FSM_R3 ;; TEMP2 = R3 XOR LSFR_A [0:7] - vpaddd %%TEMP2, %%TEMP2, %%FSM_R2 ;; TEMP2 += R2 + vpxor %%TEMP2, %%LFSR_A_LDQ, %%FSM_R3 ;; TEMP2 = R3 XOR LSFR_A [0:7] + vpaddd %%TEMP2, %%TEMP2, %%FSM_R2 ;; TEMP2 += R2 - vpxor %%TEMP1, %%TEMP1, %%TEMP1 ;; TEMP1 = 0 + vpxor %%TEMP1, %%TEMP1, %%TEMP1 ;; TEMP1 = 0 - vaesenc %%FSM_R3, %%FSM_R2, %%TEMP1 ;; R3 = AESR(R2) (encryption round key C1 = 0) - vaesenc %%FSM_R2, %%FSM_R1, %%TEMP1 ;; R2 = AESR(R1) (encryption round key C2 = 0) - vpshufb %%FSM_R1, %%TEMP2, [rel sigma] ;; R1 = sigma(TEMP2) + vaesenc %%FSM_R3, %%FSM_R2, %%TEMP1 ;; R3 = AESR(R2) (encryption round key C1 = 0) + vaesenc %%FSM_R2, %%FSM_R1, %%TEMP1 ;; R2 = AESR(R1) (encryption round key C2 = 0) + vpshufb %%FSM_R1, %%TEMP2, [rel sigma] ;; R1 = sigma(TEMP2) %endmacro ;; SNOW_V_FSM_UPDATE @@ -192,193 +198,233 @@ mksection .text ;; else: (x >> 1) ;; = (x >> 1) xor signw(inv_gA, x << 15) - ;; calculate mulx_A = (alpha*a7, ..., alpha*a0) - vpsraw %%TEMP1, %%LFSR_A_LDQ, 15 ;; 16-bit mask with sign bits preserved - vpand %%TEMP1, %%TEMP1, %%gA - vpsllw %%TEMP2, %%LFSR_A_LDQ, 1 - vpxor %%TEMP2, %%TEMP2, %%TEMP1 ;; TEMP2 = mulx_A - - ;; calculate invx_A = (alpha^-1*a15, ..., alpha^-1*a8) - vpsllw %%TEMP1, %%LFSR_A_HDQ, 15 - vpsignw %%TEMP1, %%inv_gA, %%TEMP1 ;; negate bits in inv_gA depending on LFSR_A_HDQ << 15 - vpxor %%TEMP2, %%TEMP1, %%TEMP2 - vpsrlw %%TEMP1, %%LFSR_A_HDQ, 1 - vpxor %%TEMP1, %%TEMP1, %%TEMP2 ;; TEMP1 = invx_A xor mulx_A - - ;; LFSR_A_HDQ = mulx_A XOR invx_A XOR (b7, ..., b0) XOR (a8, ..., a1) - vpalignr %%TEMP2, %%LFSR_A_HDQ, %%LFSR_A_LDQ, 2 ;; T2 = (tmpa_8, ..., tmpa_1) - vpxor %%TEMP2, %%TEMP2, %%LFSR_B_LDQ - vpxor %%T2, %%TEMP2, %%TEMP1 - - ;; calculate mulx_B - vpsraw %%TEMP1, %%LFSR_B_LDQ, 15 - vpand %%TEMP1, %%TEMP1, %%gB - vpsllw %%TEMP2, %%LFSR_B_LDQ, 1 - vpxor %%TEMP1, %%TEMP1, %%TEMP2 - - ;; T2 = mulx_B XOR (a7, ..., a0) XOR (b10, ..., b3) - vpxor %%TEMP1, %%TEMP1, %%LFSR_A_LDQ - vmovdqa %%LFSR_A_LDQ, %%LFSR_A_HDQ ;; LFSR_A_LDQ = LFSR_A_HDQ - vmovdqa %%LFSR_A_HDQ, %%T2 - - vpalignr %%TEMP2, %%LFSR_B_HDQ, %%LFSR_B_LDQ, 6 ;; (b10, ..., b3) - vmovdqa %%LFSR_B_LDQ, %%LFSR_B_HDQ ;; LFSR_B_LDQ = LFSR_B_HDQ - vpxor %%TEMP2, %%TEMP2, %%TEMP1 - - ;; calculate invx_B - vpsllw %%TEMP1, %%LFSR_B_HDQ, 15 - vpsrlw %%LFSR_B_HDQ, %%LFSR_B_HDQ, 1 - vpsignw %%TEMP1,inv_gB, %%TEMP1 - - ;; LFSR_B_HDQ = mulx_B XOR invx_B XOR (a7, ..., a0) XOR (b10, ..., b3) - vpxor %%LFSR_B_HDQ, %%LFSR_B_HDQ, %%TEMP1 - vpxor %%LFSR_B_HDQ, %%LFSR_B_HDQ, %%TEMP2 + ;; calculate mulx_A = (alpha*a7, ..., alpha*a0) + vpsraw %%TEMP1, %%LFSR_A_LDQ, 15 ;; 16-bit mask with sign bits preserved + vpand %%TEMP1, %%TEMP1, %%gA + vpsllw %%TEMP2, %%LFSR_A_LDQ, 1 + vpxor %%TEMP2, %%TEMP2, %%TEMP1 ;; TEMP2 = mulx_A + + ;; calculate invx_A = (alpha^-1*a15, ..., alpha^-1*a8) + vpsllw %%TEMP1, %%LFSR_A_HDQ, 15 + vpsignw %%TEMP1, %%inv_gA, %%TEMP1 ;; negate bits in inv_gA depending on LFSR_A_HDQ << 15 + vpxor %%TEMP2, %%TEMP1, %%TEMP2 + vpsrlw %%TEMP1, %%LFSR_A_HDQ, 1 + vpxor %%TEMP1, %%TEMP1, %%TEMP2 ;; TEMP1 = invx_A xor mulx_A + + ;; LFSR_A_HDQ = mulx_A XOR invx_A XOR (b7, ..., b0) XOR (a8, ..., a1) + vpalignr %%TEMP2, %%LFSR_A_HDQ, %%LFSR_A_LDQ, 2 ;; T2 = (tmpa_8, ..., tmpa_1) + vpxor %%TEMP2, %%TEMP2, %%LFSR_B_LDQ + vpxor %%T2, %%TEMP2, %%TEMP1 + + ;; calculate mulx_B + vpsraw %%TEMP1, %%LFSR_B_LDQ, 15 + vpand %%TEMP1, %%TEMP1, %%gB + vpsllw %%TEMP2, %%LFSR_B_LDQ, 1 + vpxor %%TEMP1, %%TEMP1, %%TEMP2 + + ;; T2 = mulx_B XOR (a7, ..., a0) XOR (b10, ..., b3) + vpxor %%TEMP1, %%TEMP1, %%LFSR_A_LDQ + vmovdqa %%LFSR_A_LDQ, %%LFSR_A_HDQ ;; LFSR_A_LDQ = LFSR_A_HDQ + vmovdqa %%LFSR_A_HDQ, %%T2 + + vpalignr %%TEMP2, %%LFSR_B_HDQ, %%LFSR_B_LDQ, 6 ;; (b10, ..., b3) + vmovdqa %%LFSR_B_LDQ, %%LFSR_B_HDQ ;; LFSR_B_LDQ = LFSR_B_HDQ + vpxor %%TEMP2, %%TEMP2, %%TEMP1 + + ;; calculate invx_B + vpsllw %%TEMP1, %%LFSR_B_HDQ, 15 + vpsrlw %%LFSR_B_HDQ, %%LFSR_B_HDQ, 1 + vpsignw %%TEMP1,inv_gB, %%TEMP1 + + ;; LFSR_B_HDQ = mulx_B XOR invx_B XOR (a7, ..., a0) XOR (b10, ..., b3) + vpxor %%LFSR_B_HDQ, %%LFSR_B_HDQ, %%TEMP1 + vpxor %%LFSR_B_HDQ, %%LFSR_B_HDQ, %%TEMP2 %endmacro ;; SNOW_V_LFSR_UPDATE ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro FUNC_START 0 +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + mov rax, rsp + sub rsp, STACK_size + and rsp, -16 + mov [rsp + _rsp_save], rax + vmovdqa [rsp + _xmm_save + 0*16], xmm6 + vmovdqa [rsp + _xmm_save + 1*16], xmm7 + vmovdqa [rsp + _xmm_save + 2*16], xmm8 + vmovdqa [rsp + _xmm_save + 3*16], xmm9 + vmovdqa [rsp + _xmm_save + 4*16], xmm10 + vmovdqa [rsp + _xmm_save + 5*16], xmm11 + vmovdqa [rsp + _xmm_save + 6*16], xmm12 + vmovdqa [rsp + _xmm_save + 7*16], xmm13 + vmovdqa [rsp + _xmm_save + 8*16], xmm14 + vmovdqa [rsp + _xmm_save + 9*16], xmm15 +%endif +%endmacro + +%macro FUNC_END 0 +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqa xmm6, [rsp + _xmm_save + 0*16] + vmovdqa xmm7, [rsp + _xmm_save + 1*16] + vmovdqa xmm8, [rsp + _xmm_save + 2*16] + vmovdqa xmm9, [rsp + _xmm_save + 3*16] + vmovdqa xmm10, [rsp + _xmm_save + 4*16] + vmovdqa xmm11, [rsp + _xmm_save + 5*16] + vmovdqa xmm12, [rsp + _xmm_save + 6*16] + vmovdqa xmm13, [rsp + _xmm_save + 7*16] + vmovdqa xmm14, [rsp + _xmm_save + 8*16] + vmovdqa xmm15, [rsp + _xmm_save + 9*16] + mov rsp, [rsp + _rsp_save] +%endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + MKGLOBAL(SNOW_V_AEAD_INIT,function,) SNOW_V_AEAD_INIT: endbranch64 + FUNC_START ;; use offset to indicate AEAD mode - mov DWORD(offset), 1 - vmovdqa LFSR_B_LDQ, [rel aead_lsfr_b_lo] - jmp snow_v_common_init + mov DWORD(offset), 1 + vmovdqa LFSR_B_LDQ, [rel aead_lsfr_b_lo] + jmp snow_v_common_init ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(SNOW_V,function,) SNOW_V: - endbranch64 - ;; use offset to indicate AEAD mode - xor DWORD(offset), DWORD(offset) - vpxor LFSR_B_LDQ, LFSR_B_LDQ, LFSR_B_LDQ + endbranch64 + FUNC_START + ;; use offset to indicate AEAD mode + xor DWORD(offset), DWORD(offset) + vpxor LFSR_B_LDQ, LFSR_B_LDQ, LFSR_B_LDQ snow_v_common_init: - ;; Init LSFR - mov rax, [job + _enc_keys] - vmovdqu LFSR_A_HDQ, [rax] - vmovdqu LFSR_B_HDQ, [rax + 16] - mov rax, [job + _iv] - vmovdqu LFSR_A_LDQ, [rax] + ;; Init LSFR + mov rax, [job + _enc_keys] + vmovdqu LFSR_A_HDQ, [rax] + vmovdqu LFSR_B_HDQ, [rax + 16] + mov rax, [job + _iv] + vmovdqu LFSR_A_LDQ, [rax] - ;; Init FSM: R1 = R2 = R3 = 0 - vpxor FSM_R1, FSM_R1, FSM_R1 - vpxor FSM_R2, FSM_R2, FSM_R2 - vpxor FSM_R3, FSM_R3, FSM_R3 + ;; Init FSM: R1 = R2 = R3 = 0 + vpxor FSM_R1, FSM_R1, FSM_R1 + vpxor FSM_R2, FSM_R2, FSM_R2 + vpxor FSM_R3, FSM_R3, FSM_R3 - vmovdqa gA, [rel alpha] - vmovdqa gB, [rel beta] - vmovdqa inv_gA, [rel alpha_inv] - vmovdqa inv_gB, [rel beta_inv] + vmovdqa gA, [rel alpha] + vmovdqa gB, [rel beta] + vmovdqa inv_gA, [rel alpha_inv] + vmovdqa inv_gB, [rel beta_inv] - mov eax, 15 + mov eax, 15 init_fsm_lfsr_loop: - SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ, LFSR_B_LDQ, LFSR_B_HDQ, \ - inv_gB, temp1, temp2, temp3, gA, gB, inv_gA - vpxor LFSR_A_HDQ, LFSR_A_HDQ, KEYSTREAM - dec eax - jnz init_fsm_lfsr_loop - - mov rax, [job + _enc_keys] - vmovdqu temp4, [rax] - vpxor FSM_R1, FSM_R1, temp4 - - SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ - inv_gB, temp1, temp2, temp3, gA, gB, inv_gA - vpxor LFSR_A_HDQ, LFSR_A_HDQ, KEYSTREAM - vmovdqu temp4, [rax + 16] - vpxor FSM_R1, FSM_R1, temp4 - - ;; At this point FSM and LSFR are initialized - - or DWORD(offset), DWORD(offset) - jz no_aead - - ;; in AEAD mode hkey = keystream_0 and endpad = keystream_1 - mov r11, [job + _snow_v_reserved] - - ;; generate hkey - SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 - vmovdqu [r11], KEYSTREAM - - ;; generate endpad - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ - inv_gB, temp1, temp2, temp3, gA, gB, inv_gA - SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 - - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ - inv_gB, temp1, temp2, temp3, gA, gB, inv_gA - - mov offset, [r11 + 24] - vmovdqu [r11 + 16], KEYSTREAM - or offset, offset - ;; if last 8 bytes endpad are not 0 skip encrypt/decrypt operation - ;; option used to calculate auth tag for decrypt and not overwrite - ;; cipher by plain when the same src/dst pointer is used - jnz no_partial_block_left + SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ, LFSR_B_LDQ, LFSR_B_HDQ, \ + inv_gB, temp1, temp2, temp3, gA, gB, inv_gA + vpxor LFSR_A_HDQ, LFSR_A_HDQ, KEYSTREAM + dec eax + jnz init_fsm_lfsr_loop + + mov rax, [job + _enc_keys] + vmovdqu temp4, [rax] + vpxor FSM_R1, FSM_R1, temp4 + + SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ + inv_gB, temp1, temp2, temp3, gA, gB, inv_gA + vpxor LFSR_A_HDQ, LFSR_A_HDQ, KEYSTREAM + vmovdqu temp4, [rax + 16] + vpxor FSM_R1, FSM_R1, temp4 + + ;; At this point FSM and LSFR are initialized + + or DWORD(offset), DWORD(offset) + jz no_aead + + ;; in AEAD mode hkey = keystream_0 and endpad = keystream_1 + mov r11, [job + _snow_v_reserved] + + ;; generate hkey + SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 + vmovdqu [r11], KEYSTREAM + + ;; generate endpad + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ + inv_gB, temp1, temp2, temp3, gA, gB, inv_gA + SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 + + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ + inv_gB, temp1, temp2, temp3, gA, gB, inv_gA + + mov offset, [r11 + 24] + vmovdqu [r11 + 16], KEYSTREAM + or offset, offset + ;; if last 8 bytes endpad are not 0 skip encrypt/decrypt operation + ;; option used to calculate auth tag for decrypt and not overwrite + ;; cipher by plain when the same src/dst pointer is used + jnz no_partial_block_left no_aead: - ;; Process input - mov r10, [job + _src] - add r10, [job + _cipher_start_src_offset_in_bytes] - mov r11, [job + _dst] - mov rax, [job + _msg_len_to_cipher_in_bytes] - xor offset, offset - ;; deal with partial block less than 16b outside main loop - and rax, 0xfffffffffffffff0 - jz final_bytes + ;; Process input + mov r10, [job + _src] + add r10, [job + _cipher_start_src_offset_in_bytes] + mov r11, [job + _dst] + mov rax, [job + _msg_len_to_cipher_in_bytes] + xor offset, offset + ;; deal with partial block less than 16b outside main loop + and rax, 0xfffffffffffffff0 + jz final_bytes encrypt_loop: - vmovdqu temp4, [r10 + offset] + vmovdqu temp4, [r10 + offset] - SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ, LFSR_B_LDQ, LFSR_B_HDQ, \ - inv_gB, temp1, temp2, temp3, gA, gB, inv_gA + SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, LFSR_A_LDQ, temp1, temp2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ, LFSR_B_LDQ, LFSR_B_HDQ, \ + inv_gB, temp1, temp2, temp3, gA, gB, inv_gA - vpxor temp4, temp4, KEYSTREAM - vmovdqu [r11 + offset], temp4 - add offset, 16 - sub rax, 16 - jnz encrypt_loop + vpxor temp4, temp4, KEYSTREAM + vmovdqu [r11 + offset], temp4 + add offset, 16 + sub rax, 16 + jnz encrypt_loop final_bytes: - mov rax, [job + _msg_len_to_cipher_in_bytes] - and rax, 0xf - jz no_partial_block_left + mov rax, [job + _msg_len_to_cipher_in_bytes] + and rax, 0xf + jz no_partial_block_left - ;; load partial block into XMM register - add r10, offset - simd_load_avx_15_1 temp4, r10, rax - SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 - vpxor temp4, temp4, KEYSTREAM - add r11, offset + ;; load partial block into XMM register + add r10, offset + simd_load_avx_15_1 temp4, r10, rax + SNOW_V_KEYSTREAM KEYSTREAM, LFSR_B_HDQ, FSM_R1, FSM_R2 + vpxor temp4, temp4, KEYSTREAM + add r11, offset - ;; use r10 and offset as temp [clobbered] - simd_store_avx_15 r11, temp4, rax, r10, offset + ;; use r10 and offset as temp [clobbered] + simd_store_avx_15 r11, temp4, rax, r10, offset no_partial_block_left: - ;; Clear registers and return data + ;; Clear registers and return data %ifdef SAFE_DATA - clear_scratch_xmms_avx_asm + clear_scratch_xmms_avx_asm %endif - mov rax, job - or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER - - ret + mov rax, job + or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER + FUNC_END + ret mksection stack-noexec diff --git a/lib/avx_t1/zuc_top_avx.c b/lib/avx_t1/zuc_top_avx.c index 5694a1089eaa1cbb33b27a5df5588763c1b35df7..33f833ad13982f42022b5944ddf9f2a4673e80a5 100644 --- a/lib/avx_t1/zuc_top_avx.c +++ b/lib/avx_t1/zuc_top_avx.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/avx_t1/zuc_x4_avx.asm b/lib/avx_t1/zuc_x4_avx.asm index 574e54ead004e5e2b08cf2169b366453c8939424..98d5e98d8c04f058ff372c1335a8c4df7d3b133a 100644 --- a/lib/avx_t1/zuc_x4_avx.asm +++ b/lib/avx_t1/zuc_x4_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2009-2022, Intel Corporation +;; Copyright (c) 2009-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/zuc_sbox.inc" -%include "include/memcpy.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/memcpy.inc" +%include "include/mb_mgr_datastruct.inc" %include "include/cet.inc" %include "include/const.inc" diff --git a/lib/avx_t2/mb_mgr_avx_t2.c b/lib/avx_t2/mb_mgr_avx_t2.c index 0e631f577ac188e71497b57b01a3fb2ed5baa397..0e9f32ba21fc95e850adee2f682610339cbe8dfb 100644 --- a/lib/avx_t2/mb_mgr_avx_t2.c +++ b/lib/avx_t2/mb_mgr_avx_t2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -48,6 +48,8 @@ #include "include/arch_sse_type1.h" /* snow3g */ #include "include/arch_sse_type2.h" /* shani */ #include "include/arch_avx_type1.h" +#include "include/arch_avx_type2.h" +#include "include/arch_avx2_type1.h" /* gcm */ #include "include/ooo_mgr_reset.h" @@ -69,6 +71,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx_t2 #define SUBMIT_HASH_BURST submit_hash_burst_avx_t2 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx_t2 +#define SET_SUITE_ID_FN set_suite_id_avx_t2 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX_T2 @@ -80,12 +83,12 @@ #define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX_T2 /* AES-GCM */ -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen2 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen2 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen2 +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen4 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen4 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen4 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen4 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen4 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen4 #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx @@ -328,7 +331,7 @@ IMB_DLL_LOCAL void init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs) { /* Check if CPU flags needed for AVX interface are present */ - if ((state->features & IMB_CPUFLAGS_AVX) != IMB_CPUFLAGS_AVX) { + if ((state->features & IMB_CPUFLAGS_AVX_T2) != IMB_CPUFLAGS_AVX_T2) { imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); return; } @@ -359,6 +362,7 @@ init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_avx; state->keyexp_192 = aes_keyexp_192_avx; @@ -370,12 +374,12 @@ init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs) state->xcbc_keyexp = aes_xcbc_expand_key_avx; state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx; - state->sha1 = sha1_avx; - state->sha224_one_block = sha224_one_block_avx; - state->sha224 = sha224_avx; - state->sha256_one_block = sha256_one_block_avx; - state->sha256 = sha256_avx; + state->sha1_one_block = sha1_one_block_sse_shani; + state->sha1 = sha1_sse_shani; + state->sha224_one_block = sha224_one_block_sse_shani; + state->sha224 = sha224_sse_shani; + state->sha256_one_block = sha256_one_block_sse_shani; + state->sha256 = sha256_sse_shani; state->sha384_one_block = sha384_one_block_avx; state->sha384 = sha384_avx; state->sha512_one_block = sha512_one_block_avx; @@ -383,6 +387,7 @@ init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_avx; state->aes128_cfb_one = aes_cfb_128_one_avx; + state->aes256_cfb_one = aes_cfb_256_one_avx; state->eea3_1_buffer = zuc_eea3_1_buffer_avx; state->eea3_4_buffer = zuc_eea3_4_buffer_avx; @@ -435,49 +440,52 @@ init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs) state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx; state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; - state->gcm128_enc = aes_gcm_enc_128_avx_gen2; - state->gcm192_enc = aes_gcm_enc_192_avx_gen2; - state->gcm256_enc = aes_gcm_enc_256_avx_gen2; - state->gcm128_dec = aes_gcm_dec_128_avx_gen2; - state->gcm192_dec = aes_gcm_dec_192_avx_gen2; - state->gcm256_dec = aes_gcm_dec_256_avx_gen2; - state->gcm128_init = aes_gcm_init_128_avx_gen2; - state->gcm192_init = aes_gcm_init_192_avx_gen2; - state->gcm256_init = aes_gcm_init_256_avx_gen2; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen2; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen2; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen2; - state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen2; - state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen2; - state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen2; - state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen2; - state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen2; - state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen2; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen2; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen2; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen2; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen2; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen2; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen2; - state->gcm128_precomp = aes_gcm_precomp_128_avx_gen2; - state->gcm192_precomp = aes_gcm_precomp_192_avx_gen2; - state->gcm256_precomp = aes_gcm_precomp_256_avx_gen2; - state->gcm128_pre = aes_gcm_pre_128_avx_gen2; - state->gcm192_pre = aes_gcm_pre_192_avx_gen2; - state->gcm256_pre = aes_gcm_pre_256_avx_gen2; - - state->ghash = ghash_avx_gen2; - state->ghash_pre = ghash_pre_avx_gen2; - - state->gmac128_init = imb_aes_gmac_init_128_avx_gen2; - state->gmac192_init = imb_aes_gmac_init_192_avx_gen2; - state->gmac256_init = imb_aes_gmac_init_256_avx_gen2; - state->gmac128_update = imb_aes_gmac_update_128_avx_gen2; - state->gmac192_update = imb_aes_gmac_update_192_avx_gen2; - state->gmac256_update = imb_aes_gmac_update_256_avx_gen2; - state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen2; - state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen2; - state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen2; + state->gcm128_enc = aes_gcm_enc_128_avx_gen4; + state->gcm192_enc = aes_gcm_enc_192_avx_gen4; + state->gcm256_enc = aes_gcm_enc_256_avx_gen4; + state->gcm128_dec = aes_gcm_dec_128_avx_gen4; + state->gcm192_dec = aes_gcm_dec_192_avx_gen4; + state->gcm256_dec = aes_gcm_dec_256_avx_gen4; + state->gcm128_init = aes_gcm_init_128_avx_gen4; + state->gcm192_init = aes_gcm_init_192_avx_gen4; + state->gcm256_init = aes_gcm_init_256_avx_gen4; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen4; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen4; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen4; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen4; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen4; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen4; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen4; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen4; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen4; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen4; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen4; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen4; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen4; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen4; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen4; + state->gcm128_precomp = aes_gcm_precomp_128_avx_gen4; + state->gcm192_precomp = aes_gcm_precomp_192_avx_gen4; + state->gcm256_precomp = aes_gcm_precomp_256_avx_gen4; + state->gcm128_pre = aes_gcm_pre_128_avx_gen4; + state->gcm192_pre = aes_gcm_pre_192_avx_gen4; + state->gcm256_pre = aes_gcm_pre_256_avx_gen4; + + state->ghash = ghash_avx_gen4; + state->ghash_pre = ghash_pre_avx_gen4; + + state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; + state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; + state->gmac256_init = imb_aes_gmac_init_256_avx_gen4; + state->gmac128_update = imb_aes_gmac_update_128_avx_gen4; + state->gmac192_update = imb_aes_gmac_update_192_avx_gen4; + state->gmac256_update = imb_aes_gmac_update_256_avx_gen4; + state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; + state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; + state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_avx; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_avx; } #include "mb_mgr_code.h" diff --git a/lib/cmake/uninstall.cmake.in b/lib/cmake/uninstall.cmake.in new file mode 100644 index 0000000000000000000000000000000000000000..7e2ca54c86661e951b517f1f3302c2d50d92f92c --- /dev/null +++ b/lib/cmake/uninstall.cmake.in @@ -0,0 +1,47 @@ +# Copyright (c) 2023, Intel Corporation +# Copyright 2000-2023 Kitware, Inc. and Contributors +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt") + message(FATAL_ERROR "No install manifest found!") +endif() + +file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" installed_files) +string(REPLACE "\n" ";" installed_files "${installed_files}") +foreach(install_file ${installed_files}) + message(STATUS "Removing ${install_file}") + if(IS_SYMLINK "${install_file}" OR EXISTS "${install_file}") + exec_program( + "@CMAKE_COMMAND@" ARGS + "-E remove \"${install_file}\"" + RETURN_VALUE retval + OUTPUT_VARIABLE out) + if(NOT "${retval}" STREQUAL 0) + message(FATAL_ERROR "Error removing ${install_file}") + endif() + else() + message(STATUS "${install_file} does not exist.") + endif() +endforeach() diff --git a/lib/cmake/unix.cmake b/lib/cmake/unix.cmake new file mode 100644 index 0000000000000000000000000000000000000000..f0fa9d8e6ce993b5a8022f30bf2a5c5394ce818c --- /dev/null +++ b/lib/cmake/unix.cmake @@ -0,0 +1,122 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# IPSec_MB library CMake Unix config +# ############################################################################## + +set(LIB IPSec_MB) # 'lib' prefix assumed on Linux + +# set compiler definitions +list(APPEND LIB_DEFINES LINUX) + +# set NASM flags +string(APPEND CMAKE_ASM_NASM_FLAGS + " -Werror -felf64 -Xgnu -gdwarf -DLINUX -D__linux__") + +# set C compiler flags +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(CMAKE_C_FLAGS + "-fPIC -W -Wall -Wextra -Wmissing-declarations \ +-Wpointer-arith -Wcast-qual -Wundef -Wwrite-strings -Wformat \ +-Wformat-security -Wunreachable-code -Wmissing-noreturn \ +-Wsign-compare -Wno-endif-labels -Wstrict-prototypes \ +-Wmissing-prototypes -Wold-style-definition \ +-fno-delete-null-pointer-checks -fwrapv -std=c99 \ +-Wno-unused-parameter -Wno-unused-variable -Wno-unused-function") +else() + set(CMAKE_C_FLAGS + "-fPIC -W -Wall -Wextra -Wmissing-declarations \ +-Wpointer-arith -Wcast-qual -Wundef -Wwrite-strings -Wformat \ +-Wformat-security -Wunreachable-code -Wmissing-noreturn \ +-Wsign-compare -Wno-endif-labels -Wstrict-prototypes \ +-Wmissing-prototypes -Wold-style-definition \ +-fno-delete-null-pointer-checks -fwrapv -std=c99") +endif() +set(CMAKE_C_FLAGS_DEBUG "-g -DDEBUG -O0") +set(CMAKE_C_FLAGS_RELEASE "-fstack-protector -D_FORTIFY_SOURCE=2 -O3") +set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now -lc") + +# -fno-strict-overflow is not supported by clang +if(CMAKE_COMPILER_IS_GNUCC) + string(APPEND CMAKE_C_FLAGS " -fno-strict-overflow") +endif() + +# set directory specific C compiler flags +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set_source_files_properties( + ${SRC_FILES_AARCH64} ${SRC_FILES_X86_64} ${SRC_FILES_NO_AESNI} + PROPERTIES COMPILE_FLAGS + "-march=armv8-a+crypto+aes+crc") +else() + set_source_files_properties( + ${SRC_FILES_AVX_T1} ${SRC_FILES_AVX_T2} + PPROPERTIES COMPILE_FLAGS + "-march=sandybridge -maes -mpclmul") + set_source_files_properties( + ${SRC_FILES_AVX2_T1} ${SRC_FILES_AVX2_T2} ${SRC_FILES_AVX2_T3} + PPROPERTIES COMPILE_FLAGS + "-march=haswell -maes -mpclmul") + set_source_files_properties( + ${SRC_FILES_AVX512_T1} ${SRC_FILES_AVX512_T2} + PROPERTIES COMPILE_FLAGS + "-march=broadwell -maes -mpclmul") + set_source_files_properties( + ${SRC_FILES_SSE_T1} ${SRC_FILES_SSE_T2} ${SRC_FILES_SSE_T3} + PROPERTIES COMPILE_FLAGS + "-march=nehalem -maes -mpclmul") + set_source_files_properties(${SRC_FILES_X86_64} + PROPERTIES COMPILE_FLAGS + "-msse4.2") + if(AESNI_EMU) + set_source_files_properties( + ${SRC_FILES_NO_AESNI} + PROPERTIES COMPILE_FLAGS + "-march=nehalem -mno-pclmul") + endif() +endif() + +######################################## +# add library target +######################################## + +add_library(${LIB} ${SRC_FILES_ASM} ${SRC_FILES_C}) + +# set library SO version +string(REPLACE "." ";" VERSION_LIST ${IPSEC_MB_VERSION}) +list(GET VERSION_LIST 0 SO_MAJOR_VER) +set_target_properties(${LIB} PROPERTIES + VERSION ${IPSEC_MB_VERSION_FULL} + SOVERSION ${SO_MAJOR_VER}) + +# set install rules +set(CMAKE_INSTALL_PREFIX "/usr" + CACHE STRING "Set default installation directory" FORCE) +install(TARGETS ${LIB} DESTINATION lib) +install(FILES ${IMB_HDR} DESTINATION include) +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/libipsec-mb.7 + ${CMAKE_CURRENT_SOURCE_DIR}/libipsec-mb-dev.7 + DESTINATION man/man7) + diff --git a/lib/cmake/windows.cmake b/lib/cmake/windows.cmake new file mode 100644 index 0000000000000000000000000000000000000000..02d39a1d890bf2134dd8687057f409c032dfbc61 --- /dev/null +++ b/lib/cmake/windows.cmake @@ -0,0 +1,119 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# IPSec_MB library CMake Windows config +# ############################################################################## + +set(LIB libIPSec_MB) +set(SRC_DEF_FILE ${CMAKE_CURRENT_BINARY_DIR}/${LIB}_lnk.def) + +if(CMAKE_GENERATOR MATCHES "Visual Studio") + if(NOT (${CMAKE_GENERATOR_PLATFORM} MATCHES "x64")) + message(FATAL_ERROR "Only 64-bit platform supported. Re-run with '-A x64' option.") + endif() +endif() + +# set NASM flags +string(APPEND CMAKE_ASM_NASM_FLAGS " -Werror -Xvc -DWIN_ABI") +set_source_files_properties(${SRC_FILES_ASM} PROPERTIES + COMPILE_FLAGS "$<$:-gcv8>") + +# set C compiler flags +set(CMAKE_C_FLAGS "/nologo /Y- /W3 /WX- /Gm- /fp:precise /EHsc /std:c11") +set(CMAKE_C_FLAGS_DEBUG "/Od /DDEBUG /Z7") +set(CMAKE_SHARED_LINKER_FLAGS "/nologo") +set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "/RELEASE /DEBUG /OPT:REF /OPT:ICF /INCREMENTAL:NO") +set(CMAKE_SHARED_LINKER_FLAGS_DEBUG "/DEBUG /INCREMENTAL:NO") +set(CMAKE_STATIC_LINKER_FLAGS "/nologo /nodefaultlib") + +# set compiler optimization flags +set_source_files_properties( + ${SRC_FILES_AVX_T1} ${SRC_FILES_AVX_T2} + ${SRC_FILES_AVX2_T1} ${SRC_FILES_AVX2_T2} + ${SRC_FILES_AVX2_T3} ${SRC_FILES_AVX512_T1} + ${SRC_FILES_AVX512_T2} + PROPERTIES COMPILE_FLAGS + "/arch:AVX $<$:/Oi /O2>") + +set_source_files_properties( + ${SRC_FILES_SSE_T1} ${SRC_FILES_SSE_T2} + ${SRC_FILES_SSE_T3} ${SRC_FILES_X86_64} + PROPERTIES COMPILE_FLAGS + "$<$:/Oi /O2>") + +# set AESNI_EMU specific compiler flags +foreach(FILE ${SRC_FILES_NO_AESNI}) + set_source_files_properties(${FILE} PROPERTIES + COMPILE_DEFINITIONS "${LIB_DEFINES}" + COMPILE_FLAGS "$<$:/Od /Oi>") +endforeach() + +# generate windows DEF file +if(NOT AESNI_EMU) + set(STR_FILTER "/c:_no_aesni") +endif() +if(NOT AVX_IFMA) + set(STR_FILTER "${STR_FILTER} /c:_avx2_t3") +endif() + +# filter unused symbol exports +if(NOT STR_FILTER) + set(GEN_DEF_FILE_CMD "copy /Y ${LIB}.def ${SRC_DEF_FILE}") +else() + set(GEN_DEF_FILE_CMD "findstr /v ${STR_FILTER} ${LIB}.def > ${SRC_DEF_FILE}") +endif() + +######################################## +# add library target +######################################## + +add_library(${LIB} ${SRC_FILES_ASM} ${SRC_FILES_C} ${SRC_DEF_FILE}) + +# set install rules +set(CMAKE_INSTALL_PREFIX "c:/Program Files" + CACHE STRING "Set default installation directory" FORCE) +install(TARGETS ${LIB} + DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_PROJECT_NAME}) +install(FILES + ${IMB_HDR} + ${SRC_DEF_FILE} + DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_PROJECT_NAME}) +if(BUILD_SHARED_LIBS) + install(FILES + $/${LIB}.exp + $/${LIB}.pdb + DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_PROJECT_NAME}) + install(FILES + $/${LIB}.dll + DESTINATION $ENV{WINDIR}/system32) +endif() + +execute_process( + COMMAND cmd /C ${GEN_DEF_FILE_CMD} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_QUIET +) + diff --git a/lib/include/aes_cbc_enc_x8_avx.inc b/lib/include/aes_cbc_enc_x8_avx.inc new file mode 100644 index 0000000000000000000000000000000000000000..3f1ee4861a8cf5ab003ee0215c0a4c341f7c0a8b --- /dev/null +++ b/lib/include/aes_cbc_enc_x8_avx.inc @@ -0,0 +1,342 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; macro to do a AES-CBC, AES-CBC-MAC and AES-XCBC +;; - multi-buffer implementation +;; - 8 buffers at a time + +;; clobbers all registers except for ARG1 and rbp +%use smartalign + +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; struct AES_ARGS { +;; void* in[8]; +;; void* out[8]; +;; UINT128* keys[8]; +;; UINT128 IV[8]; +;; } +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_cbc_enc_128_x8(AES_ARGS *args, UINT64 len); +;; arg 1: ARG : addr of AES_ARGS structure +;; arg 2: LEN : len (in units of bytes) + +struc STACK +_gpr_save: resq 8 +_len: resq 1 +endstruc + +%define GPR_SAVE_AREA rsp + _gpr_save +%define LEN_AREA rsp + _len + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rcx +%define arg4 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 rdi +%define arg4 rsi +%endif + +%define ARG arg1 +%define LEN arg2 + +%define IDX rax +%define TMP rbx + +%define KEYS0 arg3 +%define KEYS1 arg4 +%define KEYS2 rbp +%define KEYS3 r8 +%define KEYS4 r9 +%define KEYS5 r10 +%define KEYS6 r11 +%define KEYS7 r12 + +%define IN0 r13 +%define IN2 r14 +%define IN4 r15 +%define IN6 LEN + +%define XDATA0 xmm0 +%define XDATA1 xmm1 +%define XDATA2 xmm2 +%define XDATA3 xmm3 +%define XDATA4 xmm4 +%define XDATA5 xmm5 +%define XDATA6 xmm6 +%define XDATA7 xmm7 + +%define XTMP0 xmm8 +%define XTMP1 xmm9 +%define XTMP2 xmm10 +%define XTMP3 xmm11 +%define XTMP4 xmm12 +%define XTMP5 xmm13 +%define XTMP6 xmm14 +%define XTMP7 xmm15 + +%macro AES_CBC_X8 6-7 +%define %%MODE %1 ;; [in] CBC_XCBC_MAC or CBC +%define %%NROUNDS %2 ;; [in] number of AES rounds (9 - AES128, 11 - AES192, AES256 - 13) +%define %%OFFSET %3 ;; [in] numeric constant index increment +%define %%ARG_IV %4 ;; [in] pointer to array with IV pointers +%define %%ARG_KEYS %5 ;; [in] pointer to array with expanded key pointers +%define %%ARG_IN %6 ;; [in] pointer to array with input pointers (plain text) +%define %%ARG_OUT %7 ;; [in] pointer to array with destination pointers (cipher text) + + sub rsp, STACK_size + mov [GPR_SAVE_AREA + 8*0], rbp +%ifidn %%MODE, CBC_XCBC_MAC + mov [GPR_SAVE_AREA + 8*1], rbx + mov [GPR_SAVE_AREA + 8*2], r12 + mov [GPR_SAVE_AREA + 8*3], r13 + mov [GPR_SAVE_AREA + 8*4], r14 + mov [GPR_SAVE_AREA + 8*5], r15 +%ifndef LINUX + mov [GPR_SAVE_AREA + 8*6], rsi + mov [GPR_SAVE_AREA + 8*7], rdi +%endif +%endif + + xor IDX, IDX + mov [LEN_AREA], LEN + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov IN0, [%%ARG_IN + 8*0] + mov IN2, [%%ARG_IN + 8*2] + mov IN4, [%%ARG_IN + 8*4] + mov IN6, [%%ARG_IN + 8*6] + + vmovdqa XDATA0, [%%ARG_IV + 16*0] ; load IV + vmovdqa XDATA1, [%%ARG_IV + 16*1] ; load IV + vmovdqa XDATA2, [%%ARG_IV + 16*2] ; load IV + vmovdqa XDATA3, [%%ARG_IV + 16*3] ; load IV + vmovdqa XDATA4, [%%ARG_IV + 16*4] ; load IV + vmovdqa XDATA5, [%%ARG_IV + 16*5] ; load IV + vmovdqa XDATA6, [%%ARG_IV + 16*6] ; load IV + vmovdqa XDATA7, [%%ARG_IV + 16*7] ; load IV + + mov KEYS0, [%%ARG_KEYS + 8*0] + mov KEYS1, [%%ARG_KEYS + 8*1] + mov KEYS2, [%%ARG_KEYS + 8*2] + mov KEYS3, [%%ARG_KEYS + 8*3] + mov KEYS4, [%%ARG_KEYS + 8*4] + mov KEYS5, [%%ARG_KEYS + 8*5] + mov KEYS6, [%%ARG_KEYS + 8*6] + mov KEYS7, [%%ARG_KEYS + 8*7] + + ; load next block of plain text + mov TMP, [%%ARG_IN + 8*1] + vmovdqu XTMP0, [IN0 + IDX] + vmovdqu XTMP1, [TMP + IDX] + mov TMP, [%%ARG_IN + 8*3] + vmovdqu XTMP2, [IN2 + IDX] + vmovdqu XTMP3, [TMP + IDX] + mov TMP, [%%ARG_IN + 8*5] + vmovdqu XTMP4, [IN4 + IDX] + vmovdqu XTMP5, [TMP + IDX] + mov TMP, [%%ARG_IN + 8*7] + vmovdqu XTMP6, [IN6 + IDX] + vmovdqu XTMP7, [TMP + IDX] + +align 32 + +%%_main_loop: + ;; 0. ARK + vpxor XDATA0, XDATA0, [KEYS0 + 16*0] + vpxor XDATA1, XDATA1, [KEYS1 + 16*0] + vpxor XDATA2, XDATA2, [KEYS2 + 16*0] + vpxor XDATA3, XDATA3, [KEYS3 + 16*0] + vpxor XDATA4, XDATA4, [KEYS4 + 16*0] + vpxor XDATA5, XDATA5, [KEYS5 + 16*0] + vpxor XDATA6, XDATA6, [KEYS6 + 16*0] + vpxor XDATA7, XDATA7, [KEYS7 + 16*0] + + vpxor XDATA0, XDATA0, XTMP0 + vpxor XDATA1, XDATA1, XTMP1 + vpxor XDATA2, XDATA2, XTMP2 + vpxor XDATA3, XDATA3, XTMP3 + vpxor XDATA4, XDATA4, XTMP4 + vpxor XDATA5, XDATA5, XTMP5 + vpxor XDATA6, XDATA6, XTMP6 + vpxor XDATA7, XDATA7, XTMP7 + + ;; 1 to 9/11/13 ENC rounds +%assign j 1 +%rep %%NROUNDS + vaesenc XDATA0, XDATA0, [KEYS0 + 16*j] + vaesenc XDATA1, XDATA1, [KEYS1 + 16*j] + vaesenc XDATA2, XDATA2, [KEYS2 + 16*j] + vaesenc XDATA3, XDATA3, [KEYS3 + 16*j] + vaesenc XDATA4, XDATA4, [KEYS4 + 16*j] + vaesenc XDATA5, XDATA5, [KEYS5 + 16*j] + vaesenc XDATA6, XDATA6, [KEYS6 + 16*j] + vaesenc XDATA7, XDATA7, [KEYS7 + 16*j] +%assign j (j + 1) +%endrep + + ;; 1 to 9/11/13 ENCLAST round + vaesenclast XDATA0, XDATA0, [KEYS0 + 16*j] ; 10/12/14 ENC + vaesenclast XDATA1, XDATA1, [KEYS1 + 16*j] ; 10/12/14 ENC + vaesenclast XDATA2, XDATA2, [KEYS2 + 16*j] ; 10/12/14 ENC + vaesenclast XDATA3, XDATA3, [KEYS3 + 16*j] ; 10/12/14 ENC + vaesenclast XDATA4, XDATA4, [KEYS4 + 16*j] ; 10/12/14 ENC + vaesenclast XDATA5, XDATA5, [KEYS5 + 16*j] ; 10/12/14 ENC + vaesenclast XDATA6, XDATA6, [KEYS6 + 16*j] ; 10/12/14 ENC + vaesenclast XDATA7, XDATA7, [KEYS7 + 16*j] ; 10/12/14 ENC + + add IDX, %%OFFSET + cmp [LEN_AREA], IDX + jna %%_exit_main_loop + + ;; first load the next blocks into XTMP + mov TMP, [%%ARG_IN + 8*1] + vmovdqu XTMP0, [IN0 + IDX] + vmovdqu XTMP1, [TMP + IDX] + mov TMP, [%%ARG_IN + 8*3] + vmovdqu XTMP2, [IN2 + IDX] + vmovdqu XTMP3, [TMP + IDX] + mov TMP, [%%ARG_IN + 8*5] + vmovdqu XTMP4, [IN4 + IDX] + vmovdqu XTMP5, [TMP + IDX] + mov TMP, [%%ARG_IN + 8*7] + vmovdqu XTMP6, [IN6 + IDX] + vmovdqu XTMP7, [TMP + IDX] + +%ifnidn %%MODE, CBC_XCBC_MAC + sub IDX, %%OFFSET + + ;; no ciphertext write back for CBC-MAC + mov TMP, [%%ARG_OUT + 8*0] + vmovdqu [TMP + IDX], XDATA0 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*1] + vmovdqu [TMP + IDX], XDATA1 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*2] + vmovdqu [TMP + IDX], XDATA2 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*3] + vmovdqu [TMP + IDX], XDATA3 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*4] + vmovdqu [TMP + IDX], XDATA4 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*5] + vmovdqu [TMP + IDX], XDATA5 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*6] + vmovdqu [TMP + IDX], XDATA6 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*7] + vmovdqu [TMP + IDX], XDATA7 ; write back ciphertext + + add IDX, %%OFFSET +%endif + jmp %%_main_loop + +align 32 +%%_exit_main_loop: + +%ifnidn %%MODE, CBC_XCBC_MAC + ;; no ciphertext write back for CBC-MAC + sub IDX, %%OFFSET + mov TMP, [%%ARG_OUT + 8*0] + vmovdqu [TMP + IDX], XDATA0 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*1] + vmovdqu [TMP + IDX], XDATA1 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*2] + vmovdqu [TMP + IDX], XDATA2 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*3] + vmovdqu [TMP + IDX], XDATA3 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*4] + vmovdqu [TMP + IDX], XDATA4 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*5] + vmovdqu [TMP + IDX], XDATA5 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*6] + vmovdqu [TMP + IDX], XDATA6 ; write back ciphertext + mov TMP, [%%ARG_OUT + 8*7] + vmovdqu [TMP + IDX], XDATA7 ; write back ciphertext + add IDX, %%OFFSET +%endif + + ;; update IV for AES128-CBC / store digest for CBC-MAC + vmovdqa [%%ARG_IV + 16*0], XDATA0 + vmovdqa [%%ARG_IV + 16*1], XDATA1 + vmovdqa [%%ARG_IV + 16*2], XDATA2 + vmovdqa [%%ARG_IV + 16*3], XDATA3 + vmovdqa [%%ARG_IV + 16*4], XDATA4 + vmovdqa [%%ARG_IV + 16*5], XDATA5 + vmovdqa [%%ARG_IV + 16*6], XDATA6 + vmovdqa [%%ARG_IV + 16*7], XDATA7 + + ;; update IN and OUT + vmovd xmm0, [LEN_AREA] + vpshufd xmm0, xmm0, 0x44 + vpaddq xmm1, xmm0, [%%ARG_IN + 16*0] + vpaddq xmm2, xmm0, [%%ARG_IN + 16*1] + vpaddq xmm3, xmm0, [%%ARG_IN + 16*2] + vpaddq xmm4, xmm0, [%%ARG_IN + 16*3] + vmovdqa [%%ARG_IN + 16*0], xmm1 + vmovdqa [%%ARG_IN + 16*1], xmm2 + vmovdqa [%%ARG_IN + 16*2], xmm3 + vmovdqa [%%ARG_IN + 16*3], xmm4 +%ifnidn %%MODE, CBC_XCBC_MAC + vpaddq xmm5, xmm0, [%%ARG_OUT + 16*0] + vpaddq xmm6, xmm0, [%%ARG_OUT + 16*1] + vpaddq xmm7, xmm0, [%%ARG_OUT + 16*2] + vpaddq xmm8, xmm0, [%%ARG_OUT + 16*3] + vmovdqa [%%ARG_OUT + 16*0], xmm5 + vmovdqa [%%ARG_OUT + 16*1], xmm6 + vmovdqa [%%ARG_OUT + 16*2], xmm7 + vmovdqa [%%ARG_OUT + 16*3], xmm8 +%endif + + ;; XMMs are saved at a higher level + mov rbp, [GPR_SAVE_AREA + 8*0] +%ifidn %%MODE, CBC_XCBC_MAC + mov rbx, [GPR_SAVE_AREA + 8*1] + mov r12, [GPR_SAVE_AREA + 8*2] + mov r13, [GPR_SAVE_AREA + 8*3] + mov r14, [GPR_SAVE_AREA + 8*4] + mov r15, [GPR_SAVE_AREA + 8*5] +%ifndef LINUX + mov rsi, [GPR_SAVE_AREA + 8*6] + mov rdi, [GPR_SAVE_AREA + 8*7] +%endif +%endif + + add rsp, STACK_size + +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif ;; SAFE_DATA + +%endmacro + +mksection stack-noexec diff --git a/lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc b/lib/include/aes_cntr_by16_vaes_avx512.inc similarity index 99% rename from lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc rename to lib/include/aes_cntr_by16_vaes_avx512.inc index 8e859aaae1b9d894a50fcba6866b71c1e584965c..af33dfca513413ad19404a5ed42cc97c9bd1321d 100644 --- a/lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc +++ b/lib/include/aes_cntr_by16_vaes_avx512.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2019-2022, Intel Corporation All rights reserved. +; Copyright(c) 2019-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -30,14 +30,14 @@ %ifndef AES_CNTR_VAES_AVX512_INC %define AES_CNTR_VAES_AVX512_INC -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" -%include "include/aes_common.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" +%include "include/aes_common.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/include/aes_common.asm b/lib/include/aes_common.inc similarity index 99% rename from lib/include/aes_common.asm rename to lib/include/aes_common.inc index f0410deaa1071f45e3d3f6af0faa5f285c351d7b..0d0279ad246985a7e6489ae3c6965f8918f58d85 100644 --- a/lib/include/aes_common.asm +++ b/lib/include/aes_common.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef _AES_COMMON_ASM_ -%define _AES_COMMON_ASM_ +%ifndef _AES_COMMON_INC_ +%define _AES_COMMON_INC_ -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" ;; ============================================================================= ;; Generic macro to produce code that executes %%OPCODE instruction @@ -1083,7 +1083,7 @@ %endmacro -%endif ;; _AES_COMMON_ASM +%endif ;; _AES_COMMON_INC_ ;;; =========================================================================== ;;; Handles AES decryption rounds for 0 to 8 blocks on SSE diff --git a/lib/include/aesni_emu.h b/lib/include/aesni_emu.h index e96f524f05d28af7c4d592c5b3efd3cf9b7530d1..4c5ef3a07377adf3fee0b73dc3c568577f13a99b 100644 --- a/lib/include/aesni_emu.h +++ b/lib/include/aesni_emu.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/aesni_emu.inc b/lib/include/aesni_emu.inc index ceee77ba12193a40d167ed81db40ca88b96f1874..30d0eec3b3a99d50b994460cd2c0437c42e4c192 100644 --- a/lib/include/aesni_emu.inc +++ b/lib/include/aesni_emu.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,7 +29,7 @@ %ifndef _AESNI_EMU_INC_ %define _AESNI_EMU_INC_ -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Utility macros and defines to assist AESNI translation macros diff --git a/lib/include/arch_aarch64.h b/lib/include/arch_aarch64.h new file mode 100644 index 0000000000000000000000000000000000000000..cac79a48055c57c760839f588800ba7725b879a7 --- /dev/null +++ b/lib/include/arch_aarch64.h @@ -0,0 +1,67 @@ +/********************************************************************** + Copyright(c) 2023 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#ifndef ARCH_AARCH64_H +#define ARCH_AARCH64_H + +#include "ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* moved from MB MGR */ + +IMB_JOB *submit_job_zuc_eea3_aarch64(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_aarch64(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_aarch64(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_aarch64(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_aarch64(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_aarch64(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_aarch64(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_aarch64(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); +IMB_JOB *submit_job_snow3g_uea2_aarch64(IMB_MGR *state, IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uea2_aarch64(IMB_MGR *state); + +IMB_JOB *submit_job_snow3g_uia2_aarch64(MB_MGR_SNOW3G_OOO *state,IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uia2_aarch64(MB_MGR_SNOW3G_OOO *state); + +IMB_DLL_EXPORT void set_suite_id_aarch64(IMB_MGR *state, IMB_JOB *job); + +uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); + +#endif /* ARCH_AARCH64_H */ diff --git a/lib/include/arch_aarch64_noaesni.h b/lib/include/arch_aarch64_noaesni.h new file mode 100644 index 0000000000000000000000000000000000000000..f995ba3a2033e2f1e1da708a6844fc5351efdcda --- /dev/null +++ b/lib/include/arch_aarch64_noaesni.h @@ -0,0 +1,67 @@ +/********************************************************************** + Copyright(c) 2023 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#ifndef ARCH_AARCH64_NOAESNI_H +#define ARCH_AARCH64_NOAESNI_H + +#include "ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* moved from MB MGR */ + +IMB_JOB *submit_job_zuc_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_aarch64_no_aesni(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); +IMB_JOB *submit_job_snow3g_uea2_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uea2_aarch64_no_aesni(IMB_MGR *state); + +IMB_JOB *submit_job_snow3g_uia2_aarch64_no_aesni(MB_MGR_SNOW3G_OOO *state,IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uia2_aarch64_no_aesni(MB_MGR_SNOW3G_OOO *state); + +IMB_DLL_EXPORT void set_suite_id_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *job); + +uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); + +#endif /* ARCH_AARCH64_NOAESNI_H */ diff --git a/lib/include/arch_aarch64_sve256.h b/lib/include/arch_aarch64_sve256.h new file mode 100644 index 0000000000000000000000000000000000000000..69c7714c4d63b82b9a26fff21ac1d026596faf4b --- /dev/null +++ b/lib/include/arch_aarch64_sve256.h @@ -0,0 +1,67 @@ +/********************************************************************** + Copyright(c) 2023 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#ifndef ARCH_AARCH64_SVE256_H +#define ARCH_AARCH64_SVE256_H + +#include "ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* moved from MB MGR */ + +IMB_JOB *submit_job_zuc_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_aarch64_sve256(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_aarch64_sve256(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); +IMB_JOB *submit_job_snow3g_uea2_aarch64_sve256(IMB_MGR *state, IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uea2_aarch64_sve256(IMB_MGR *state); + +IMB_JOB *submit_job_snow3g_uia2_aarch64_sve256(MB_MGR_SNOW3G_OOO *state,IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uia2_aarch64_sve256(MB_MGR_SNOW3G_OOO *state); + +IMB_DLL_EXPORT void set_suite_id_aarch64_sve256(IMB_MGR *state, IMB_JOB *job); + +uint32_t crc32_wimax_ofdma_data_aarch64(const void *msg, const uint64_t len); + +#endif /* ARCH_AARCH64_SVE256_H */ diff --git a/lib/include/arch_avx2_type1.h b/lib/include/arch_avx2_type1.h index 874cd0db9215d59ebfe04d3cd79683aa78c9b1ca..97263599e79c76f3c78cced4d74437aa68ed7c51 100644 --- a/lib/include/arch_avx2_type1.h +++ b/lib/include/arch_avx2_type1.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -38,6 +38,30 @@ void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); void call_sha256_oct_avx2_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); void call_sha512_x4_avx2_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); +IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest); +IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest); +IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest); +IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest); +IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest); + +/*AES-CFB*/ +IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); +IMB_DLL_EXPORT void aes_cfb_256_one_avx2(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); + /* moved from MB MGR */ IMB_JOB *submit_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state, IMB_JOB *job); @@ -112,4 +136,6 @@ IMB_JOB *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); +IMB_DLL_EXPORT void set_suite_id_avx2_t1(IMB_MGR *state, IMB_JOB *job); + #endif /* IMB_ASM_AVX2_T1_H */ diff --git a/lib/include/arch_avx2_type2.h b/lib/include/arch_avx2_type2.h index 47ec7d907b71f7fb331e596cecc4351b1b2fd4f6..02241d0dfa1d7a71c5afb6d3bc83a40ed0b9cfc9 100644 --- a/lib/include/arch_avx2_type2.h +++ b/lib/include/arch_avx2_type2.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -48,4 +48,26 @@ void aes_ecb_dec_192_vaes_avx2(const void *in, const void *keys, void aes_ecb_dec_128_vaes_avx2(const void *in, const void *keys, void *out, uint64_t len_bytes); +/* moved from MB MGR */ +IMB_JOB *submit_job_zuc_eea3_gfni_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_gfni_avx2(MB_MGR_ZUC_OOO *state); + +IMB_JOB *flush_job_zuc256_eea3_gfni_avx2(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_gfni_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); + +IMB_JOB *submit_job_zuc_eia3_gfni_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_gfni_avx2(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_gfni_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_gfni_avx2(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +IMB_DLL_EXPORT void set_suite_id_avx2_t2(IMB_MGR *state, IMB_JOB *job); + #endif /* IMB_ASM_AVX2_T2_H */ diff --git a/lib/include/arch_avx2_type3.h b/lib/include/arch_avx2_type3.h new file mode 100644 index 0000000000000000000000000000000000000000..91f451a642a32434f6d2e49b6d1662875d830df1 --- /dev/null +++ b/lib/include/arch_avx2_type3.h @@ -0,0 +1,40 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* AVX_IFMA */ + +#ifndef IMB_ASM_AVX2_T3_H +#define IMB_ASM_AVX2_T3_H + +#include "ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +void poly1305_mac_fma_avx2(IMB_JOB *job); + +IMB_DLL_EXPORT void set_suite_id_avx2_t3(IMB_MGR *state, IMB_JOB *job); + +#endif /* IMB_ASM_AVX2_T3_H */ diff --git a/lib/include/arch_avx512_type1.h b/lib/include/arch_avx512_type1.h index 523ba6d8055249765db91a0631821aa3d481d629..700cb65d2ce5258dce19176ae4315a444a119d5e 100644 --- a/lib/include/arch_avx512_type1.h +++ b/lib/include/arch_avx512_type1.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -156,6 +156,31 @@ void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); void call_sha512_x8_avx512_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); -#endif /* IMB_ASM_AVX512_T1_H */ +IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest); +IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest); +IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest); +IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest); +IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest); + +/* AES-CFB */ +IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); +IMB_DLL_EXPORT void aes_cfb_256_one_avx512(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); + +IMB_DLL_EXPORT void set_suite_id_avx512_t1(IMB_MGR *state, IMB_JOB *job); +#endif /* IMB_ASM_AVX512_T1_H */ diff --git a/lib/include/arch_avx512_type2.h b/lib/include/arch_avx512_type2.h index c1170698c17bae39aa70a590c8c74e122117aebc..af7afdce031fb81712693ab506979f36603ba331 100644 --- a/lib/include/arch_avx512_type2.h +++ b/lib/include/arch_avx512_type2.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -71,6 +71,11 @@ void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); +void aes_ecb_quic_enc_128_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t num_buffers); +void aes_ecb_quic_enc_256_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t num_buffers); + /* AES-CBCS */ void aes_cbcs_1_9_dec_128_vaes_avx512(const void *in, const uint8_t *IV, const void *keys, void *out, @@ -198,6 +203,6 @@ submit_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB * flush_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state); +IMB_DLL_EXPORT void set_suite_id_avx512_t2(IMB_MGR *state, IMB_JOB *job); #endif /* IMB_ASM_AVX512_T2_H */ - diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index c0a8812fedd89ec26ea8cf7a3528cf0a96c1285c..843b0508f099916a95838871f353b56333f4f93d 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -81,13 +81,10 @@ void aes_ecb_dec_192_avx(const void *in, const void *keys, void aes_ecb_dec_128_avx(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_256_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); - +void aes_ecb_quic_enc_128_avx(const void *in, const void *keys, + void *out, uint64_t num_buffers); +void aes_ecb_quic_enc_256_avx(const void *in, const void *keys, + void *out, uint64_t num_buffers); /* AES128-ECBENC */ void aes128_ecbenc_x3_avx(const void *in, void *keys, void *out1, void *out2, void *out3); @@ -97,6 +94,14 @@ void aes_cbcs_1_9_dec_128_avx(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes, void *next_iv); +/* AES-CFB */ +IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); +IMB_DLL_EXPORT void aes_cfb_256_one_avx(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); + /* stitched AES128-CNTR, CRC32 and BIP */ IMB_JOB *submit_job_pon_enc_avx(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_avx(IMB_JOB *job); @@ -250,4 +255,22 @@ void call_sha1_mult_avx_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); void call_sha_256_mult_avx_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); void call_sha512_x2_avx_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); +IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest); +IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest); +IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest); +IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest); +IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest); + +IMB_DLL_EXPORT void set_suite_id_avx_t1(IMB_MGR *state, IMB_JOB *job); + #endif /* IMB_ASM_AVX_T1_H */ diff --git a/lib/include/arch_avx_type2.h b/lib/include/arch_avx_type2.h new file mode 100644 index 0000000000000000000000000000000000000000..ec48e003606691181b8ea470a7f747d3863ba6c5 --- /dev/null +++ b/lib/include/arch_avx_type2.h @@ -0,0 +1,37 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* AVX TYPE1 + SHANI + GFNI */ + +#ifndef IMB_ASM_AVX_T2_H +#define IMB_ASM_AVX_T2_H + +#include "ipsec-mb.h" + +IMB_DLL_EXPORT void set_suite_id_avx_t2(IMB_MGR *state, IMB_JOB *job); + +#endif /* IMB_ASM_AVX_T2_H */ diff --git a/lib/include/arch_noaesni.h b/lib/include/arch_noaesni.h index 7edb9d2c41dc3674ca4c8ed6acb5dbb60e5df98d..6341e3f80082c930eeae3f88121f39ff775dff86 100644 --- a/lib/include/arch_noaesni.h +++ b/lib/include/arch_noaesni.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -83,6 +83,11 @@ void aes_ecb_dec_192_sse_no_aesni(const void *in, const void *keys, void aes_ecb_dec_128_sse_no_aesni(const void *in, const void *keys, void *out, uint64_t len_bytes); +void aes_ecb_quic_enc_128_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t num_buffers); +void aes_ecb_quic_enc_256_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t num_buffers); + /* AES128-ECBENC */ void aes128_ecbenc_x3_sse_no_aesni(const void *in, void *keys, void *out1, void *out2, void *out3); @@ -186,6 +191,8 @@ uint32_t ethernet_fcs_sse_no_aesni_local(const void *msg, const uint64_t len, const void *tag_ouput); +IMB_DLL_EXPORT void set_suite_id_sse_no_aesni(IMB_MGR *state, IMB_JOB *job); + #endif /* IMB_ARCH_NOAESNI_H */ diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index a74702c7dcd9057b5e3e612784a019d14a4c467c..ac915f103073808a3aeb4b104c582e889c79c5b4 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -82,6 +82,12 @@ void aes_ecb_dec_192_by4_sse(const void *in, const void *keys, void aes_ecb_dec_128_by4_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); +void aes_ecb_quic_enc_128_sse(const void *in, const void *keys, + void *out, uint64_t num_buffers); +void aes_ecb_quic_enc_256_sse(const void *in, const void *keys, + void *out, uint64_t num_buffers); + + /* AES128-ECBENC */ void aes128_ecbenc_x3_sse(const void *in, void *keys, void *out1, void *out2, void *out3); @@ -91,6 +97,14 @@ void aes_cbcs_1_9_dec_128_sse(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes, void *next_iv); +/* AES-CFB */ +IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); +IMB_DLL_EXPORT void aes_cfb_256_one_sse(void *out, const void *in, + const void *iv, const void *keys, + uint64_t len); + /* stitched AES128-CNTR, CRC32 and BIP */ IMB_JOB *submit_job_pon_enc_sse(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_sse(IMB_JOB *job); @@ -119,6 +133,22 @@ void call_sha1_mult_sse_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); void call_sha_256_mult_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); void call_sha512_x2_sse_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); +IMB_DLL_EXPORT void sha1_sse(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest); +IMB_DLL_EXPORT void sha224_sse(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest); +IMB_DLL_EXPORT void sha256_sse(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest); +IMB_DLL_EXPORT void sha384_sse(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest); +IMB_DLL_EXPORT void sha512_sse(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest); + /* moved from MB MGR */ IMB_JOB *submit_job_aes128_enc_sse(MB_MGR_AES_OOO *state, IMB_JOB *job); @@ -246,4 +276,6 @@ IMB_JOB *submit_job_snow3g_uia2_sse(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_snow3g_uia2_sse(MB_MGR_SNOW3G_OOO *state); +IMB_DLL_EXPORT void set_suite_id_sse_t1(IMB_MGR *state, IMB_JOB *job); + #endif /* IMB_ARCH_SSE_TYPE1_H */ diff --git a/lib/include/arch_sse_type2.h b/lib/include/arch_sse_type2.h index 7910132b5071a3346d9d62ae34e79705539b9bb3..77c66403890d0b264f8ebbb8d543fb0ec71b9f00 100644 --- a/lib/include/arch_sse_type2.h +++ b/lib/include/arch_sse_type2.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -38,6 +38,18 @@ void call_sha1_ni_x2_sse_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); void call_sha224_ni_x2_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +IMB_DLL_EXPORT void sha1_sse_shani(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha1_one_block_sse_shani(const void *data, void *digest); + +IMB_DLL_EXPORT void sha224_sse_shani(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha224_one_block_sse_shani(const void *data, void *digest); + +IMB_DLL_EXPORT void sha256_sse_shani(const void *data, const uint64_t length, + void *digest); +IMB_DLL_EXPORT void sha256_one_block_sse_shani(const void *data, void *digest); + /* Moved from MB MGR */ IMB_JOB *submit_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state, @@ -64,4 +76,6 @@ IMB_JOB *submit_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_DLL_EXPORT void set_suite_id_sse_t2(IMB_MGR *state, IMB_JOB *job); + #endif /* IMB_ARCH_SSE_TYPE2_H */ diff --git a/lib/include/arch_sse_type3.h b/lib/include/arch_sse_type3.h index 0eaae1dd1eb18b95c92fe0b0e85ba13e99dd0c22..02df33aaf276806072ba52a2ae0b1daf69a23fc1 100644 --- a/lib/include/arch_sse_type3.h +++ b/lib/include/arch_sse_type3.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -112,4 +112,6 @@ IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); +IMB_DLL_EXPORT void set_suite_id_sse_t3(IMB_MGR *state, IMB_JOB *job); + #endif /* IMB_ASM_SSE_T3_H */ diff --git a/lib/include/arch_x86_64.h b/lib/include/arch_x86_64.h index bb21f7feaf46148b7816fba92ec9df18e4a5a5fd..87d4a9bcde3f14559b582b2c6d62da5f49b17222 100644 --- a/lib/include/arch_x86_64.h +++ b/lib/include/arch_x86_64.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -132,4 +132,15 @@ void docsis_des_dec_basic(const void *input, void *output, const int size, */ IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr); +/** + * @brief Atomic 64-bit counter increment + * + * This implements counter post increment. + * + * @param counter pointer to a 64-bit counter + * + * @return Counter value prior to increment + */ +IMB_DLL_LOCAL uint64_t atomic_uint64_inc(uint64_t *counter); + #endif /* IMB_ARCH_X86_64_H */ diff --git a/lib/include/asm.h b/lib/include/asm.h deleted file mode 100644 index 3d2cbed5c52263e253ca6e01dbe86ad29e211d2f..0000000000000000000000000000000000000000 --- a/lib/include/asm.h +++ /dev/null @@ -1,248 +0,0 @@ -/******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -/* interface to asm routines */ - -#ifndef IMB_ASM_H -#define IMB_ASM_H - -#include "ipsec-mb.h" -#include "ipsec_ooo_mgr.h" - -/* Define interface to base asm code */ - -/* AES-CBC */ -void aes_cbc_enc_128_x8(AES_ARGS *args, uint64_t len_in_bytes); -void aes_cbc_enc_192_x8(AES_ARGS *args, uint64_t len_in_bytes); -void aes_cbc_enc_256_x8(AES_ARGS *args, uint64_t len_in_bytes); - -void aes_cbc_dec_128_avx(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_192_avx(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_256_avx(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); - -void aes_cbc_dec_128_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_192_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_256_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); - -void aes_cbc_dec_128_sse(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_192_sse(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_256_sse(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); - -void aes_cbc_dec_128_by8_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, uint64_t len_bytes); -void aes_cbc_dec_192_by8_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, uint64_t len_bytes); -void aes_cbc_dec_256_by8_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, uint64_t len_bytes); - -void aes_cbc_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_192_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_256_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); - -/* AES-CTR */ -void aes_cntr_256_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_192_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_128_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); - -void aes_cntr_256_sse_no_aesni(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_192_sse_no_aesni(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_128_sse_no_aesni(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); - - -void aes_cntr_256_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_192_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_128_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); - -void aes_cntr_128_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_192_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_256_submit_vaes_avx512(IMB_JOB *job); - -/* AES-CTR-BITLEN */ -void aes_cntr_bit_256_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_192_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_128_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); - -void aes_cntr_bit_256_sse_no_aesni(const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_192_sse_no_aesni(const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_128_sse_no_aesni(const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bits, uint64_t IV_len); - -void aes_cntr_bit_256_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_192_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_128_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); - -void aes_cntr_bit_128_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_bit_192_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_bit_256_submit_vaes_avx512(IMB_JOB *job); - -/* AES-CCM */ -IMB_JOB *aes_cntr_ccm_128_sse(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_256_sse(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_128_sse_no_aesni(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_256_sse_no_aesni(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_128_avx(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_256_avx(IMB_JOB *job); - -/* AES-ECB */ -void aes_ecb_enc_256_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -void aes_ecb_enc_256_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); - - -void aes_ecb_enc_256_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -void aes_ecb_dec_256_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -void aes_ecb_dec_256_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); - - -void aes_ecb_dec_256_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -/* AES128-ECBENC */ -void aes128_ecbenc_x3_sse(const void *in, void *keys, - void *out1, void *out2, void *out3); -void aes128_ecbenc_x3_sse_no_aesni(const void *in, void *keys, - void *out1, void *out2, void *out3); -void aes128_ecbenc_x3_avx(const void *in, void *keys, - void *out1, void *out2, void *out3); - -/* AES-CBCS */ -void aes_cbcs_1_9_dec_128_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); -void aes_cbcs_1_9_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); -void aes_cbcs_1_9_dec_128_avx(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); -void aes_cbcs_1_9_dec_128_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); - - -/* stitched AES128-CNTR, CRC32 and BIP */ -IMB_JOB *submit_job_pon_enc_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_avx(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_sse(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_sse(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_sse_no_aesni(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_sse_no_aesni(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_vaes_avx512(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_vaes_avx512(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_no_ctr_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_avx(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_no_ctr_sse(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_sse(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_no_ctr_sse_no_aesni(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_sse_no_aesni(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_no_ctr_vaes_avx512(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_vaes_avx512(IMB_JOB *job); - -#endif /* IMB_ASM_H */ - - diff --git a/lib/include/cet.inc b/lib/include/cet.inc index b8e9dde853919e5a73cabb468bb18a86468bbd87..a89fd94e80e23762469f40b7226949c74efc7106 100644 --- a/lib/include/cet.inc +++ b/lib/include/cet.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/chacha20_poly1305.h b/lib/include/chacha20_poly1305.h index 233689c6e7745b9a5685eb0f729c6eea8f9a8d66..e1b3c1ffc420d618c50a370adaf4c353eba44680 100644 --- a/lib/include/chacha20_poly1305.h +++ b/lib/include/chacha20_poly1305.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -78,6 +78,12 @@ void poly1305_aead_update_fma_avx512(const void *msg, const uint64_t msg_len, void *hash, const void *key); void poly1305_aead_complete_fma_avx512(const void *hash, const void *key, void *tag); +#ifdef AVX_IFMA +void poly1305_aead_update_fma_avx2(const void *msg, const uint64_t msg_len, + void *hash, const void *key); +void poly1305_aead_complete_fma_avx2(const void *hash, const void *key, + void *tag); +#endif void gen_keystr_poly_key_sse(const void *key, const void *iv, const uint64_t len, void *ks); @@ -93,6 +99,16 @@ void init_chacha20_poly1305_avx(const void *key, struct chacha20_poly1305_context_data *ctx, const void *iv, const void *aad, const uint64_t aad_len); +void init_chacha20_poly1305_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + const void *iv, const void *aad, + const uint64_t aad_len); +#ifdef AVX_IFMA +void init_chacha20_poly1305_fma_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + const void *iv, const void *aad, + const uint64_t aad_len); +#endif void init_chacha20_poly1305_avx512(const void *key, struct chacha20_poly1305_context_data *ctx, const void *iv, const void *aad, @@ -113,6 +129,12 @@ void update_enc_chacha20_poly1305_avx2(const void *key, struct chacha20_poly1305_context_data *ctx, void *dst, const void *src, const uint64_t len); +#ifdef AVX_IFMA +void update_enc_chacha20_poly1305_fma_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + void *dst, const void *src, + const uint64_t len); +#endif void update_enc_chacha20_poly1305_avx512(const void *key, struct chacha20_poly1305_context_data *ctx, void *dst, const void *src, @@ -133,6 +155,12 @@ void update_dec_chacha20_poly1305_avx2(const void *key, struct chacha20_poly1305_context_data *ctx, void *dst, const void *src, const uint64_t len); +#ifdef AVX_IFMA +void update_dec_chacha20_poly1305_fma_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + void *dst, const void *src, + const uint64_t len); +#endif void update_dec_chacha20_poly1305_avx512(const void *key, struct chacha20_poly1305_context_data *ctx, void *dst, const void *src, @@ -145,6 +173,13 @@ void finalize_chacha20_poly1305_sse(struct chacha20_poly1305_context_data *ctx, void *tag, const uint64_t tag_len); void finalize_chacha20_poly1305_avx(struct chacha20_poly1305_context_data *ctx, void *tag, const uint64_t tag_len); +void finalize_chacha20_poly1305_avx2(struct chacha20_poly1305_context_data *ctx, + void *tag, const uint64_t tag_len); +#ifdef AVX_IFMA +void +finalize_chacha20_poly1305_fma_avx2(struct chacha20_poly1305_context_data *ctx, + void *tag, const uint64_t tag_len); +#endif void finalize_chacha20_poly1305_avx512( struct chacha20_poly1305_context_data *ctx, void *tag, const uint64_t tag_len); diff --git a/lib/include/chacha_poly_defines.asm b/lib/include/chacha_poly_defines.inc similarity index 93% rename from lib/include/chacha_poly_defines.asm rename to lib/include/chacha_poly_defines.inc index 445c0bacf73c2a1dffcaa45eae9aa4153ddc56c3..793b87584cf5196736aee119308bc5252c21c064 100644 --- a/lib/include/chacha_poly_defines.asm +++ b/lib/include/chacha_poly_defines.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,8 +25,8 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef CHACHA_POLY_DEFINES_ASM_INCLUDED -%define CHACHA_POLY_DEFINES_ASM_INCLUDED +%ifndef CHACHA_POLY_DEFINES_INC_INCLUDED +%define CHACHA_POLY_DEFINES_INC_INCLUDED ;;define the fields of gcm_context_data struct ;; struct chacha20_poly1305_context_data { @@ -55,4 +55,4 @@ %define RemainCtBytes (8*21) %define IV (8*22) -%endif ; CHACHA_POLY_DEFINES_ASM_INCLUDED +%endif ; CHACHA_POLY_DEFINES_INC_INCLUDED diff --git a/lib/include/clear_regs.asm b/lib/include/clear_regs.inc similarity index 87% rename from lib/include/clear_regs.asm rename to lib/include/clear_regs.inc index 3df53f47a59d425f3f4f0413ec6e3a61dc39187f..a5d5f5a94b6c43b51e46999efa14aaabcaf4c6e9 100644 --- a/lib/include/clear_regs.asm +++ b/lib/include/clear_regs.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef _CLEAR_REGS_ASM_ -%define _CLEAR_REGS_ASM_ +%ifndef _CLEAR_REGS_INC_ +%define _CLEAR_REGS_INC_ -%include "include/os.asm" +%include "include/os.inc" ; ; This macro clears any GP registers passed @@ -63,6 +63,39 @@ %endrep %endmacro +; +; This macro clears any ZMM registers passed on +; +%macro clear_zmms_avx512 1-32 + +%assign num_reg %0 +%assign num_xors 3 + +; clear up to 3 registers with vpxorq +%if num_reg < 3 +%assign num_xors num_reg +%endif + +; save 1st register for later +%xdefine %%SAVE_XMM XWORD(%1) + +%rep num_xors + vpxorq XWORD(%1), XWORD(%1), XWORD(%1) +%rotate 1 +%endrep + +; clear the rest of the registers with move from the 1st register +%assign num_reg (num_reg - num_xors) + +%rep num_reg + vmovdqa64 XWORD(%1), %%SAVE_XMM +%rotate 1 +%endrep + +%undef %%SAVE_XMM + +%endmacro + ; ; This macro clears scratch GP registers ; for Windows or Linux @@ -222,4 +255,4 @@ vzeroupper %endmacro -%endif ;; _CLEAR_REGS_ASM +%endif ;; _CLEAR_REGS_INC_ diff --git a/lib/include/clear_regs_mem.h b/lib/include/clear_regs_mem.h index 8b43f5177e9321cb93c3e65a402eb9a8882e1f56..4b9238182328c823696bc866665d16fc2edd69ae 100644 --- a/lib/include/clear_regs_mem.h +++ b/lib/include/clear_regs_mem.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/const.inc b/lib/include/const.inc index 57c475e547d19b4571ddb2bda3a3369c3cdb9f7b..ca3160edeb52028dfb20cf0b972fcdaa68282e91 100644 --- a/lib/include/const.inc +++ b/lib/include/const.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/constant_lookup.h b/lib/include/constant_lookup.h index 9336833a529f89b212bc4ad422bd5b74902842f0..d95c34e175324930ae3c4b203865ed2434af80cd 100644 --- a/lib/include/constant_lookup.h +++ b/lib/include/constant_lookup.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/constant_lookup.asm b/lib/include/constant_lookup.inc similarity index 98% rename from lib/include/constant_lookup.asm rename to lib/include/constant_lookup.inc index a0ce4ff6905c7ee6ff60d54d1b2c8f4571aa9740..4b9f19c3d98459c0a82b6084030e857ab5d75f8b 100644 --- a/lib/include/constant_lookup.asm +++ b/lib/include/constant_lookup.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,6 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; +%ifndef _CONSTANT_LOOKUP_INC_ +%define _CONSTANT_LOOKUP_INC_ + extern idx_rows_avx512 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -175,3 +178,5 @@ extern idx_rows_avx512 %%MAP_TAB_0, %%MAP_TAB_1, \ %%MAP_TAB_2, %%MAP_TAB_3, %%KR1 %endmacro + +%endif ;; _CONSTANT_LOOKUP_INC_ diff --git a/lib/include/constants.h b/lib/include/constants.h index fd1b342eb7cac43c9c1c942e25c4a2531c3c65b8..91b21afaef8eecec9dcff33f36b5dbbe3e2c0131 100644 --- a/lib/include/constants.h +++ b/lib/include/constants.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/constants.asm b/lib/include/constants.inc similarity index 98% rename from lib/include/constants.asm rename to lib/include/constants.inc index dbb7c19a6ed2e2b1584c41f8d212a7cfa79611ea..152c1ca3b10c62af34b73da72e7f54d54d6ca0c8 100644 --- a/lib/include/constants.asm +++ b/lib/include/constants.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/cpu_feature.h b/lib/include/cpu_feature.h index 7084a6ccadec34cb7aa6b58973ce6f066ede6c36..9620588f3d449a9039406dfd76a463a731e69428 100644 --- a/lib/include/cpu_feature.h +++ b/lib/include/cpu_feature.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/crc32.inc b/lib/include/crc32.inc index cebdb407eab7f3321bc2ab6ead08682bf0fae310..80cf2d1454bf18d0d009868fc5230d2b093fa140 100644 --- a/lib/include/crc32.inc +++ b/lib/include/crc32.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/crc32_const.inc b/lib/include/crc32_const.inc index 9bb2b6e4760ae01183896827e18dde8a9bec3a1b..9977d5e1380aab8763ab01a3540e5295be086792 100644 --- a/lib/include/crc32_const.inc +++ b/lib/include/crc32_const.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/crc32_refl.inc b/lib/include/crc32_refl.inc index 5439a7ac35b668d091a8a8e8069039c6fb11577e..4dd77fa7def8ff95695e455826c5b829f6089adb 100644 --- a/lib/include/crc32_refl.inc +++ b/lib/include/crc32_refl.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/crc32_refl_const.inc b/lib/include/crc32_refl_const.inc index 5f5e466ef9f0f3f6317cc5e4f2321c66226528c7..ff05e12d90e32df90bde4469ee93b2e014413fec 100644 --- a/lib/include/crc32_refl_const.inc +++ b/lib/include/crc32_refl_const.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/datastruct.asm b/lib/include/datastruct.inc similarity index 97% rename from lib/include/datastruct.asm rename to lib/include/datastruct.inc index 43a49d225d15fb0d60d4dfbec8b5285f6375046c..c5afc136507ab9f463414d7c061dfce433e3055f 100644 --- a/lib/include/datastruct.asm +++ b/lib/include/datastruct.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -92,8 +92,8 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%ifndef _DATASTRUCT_ASM_ -%define _DATASTRUCT_ASM_ +%ifndef _DATASTRUCT_INC_ +%define _DATASTRUCT_INC_ ;; START_FIELDS %macro START_FIELDS 0 @@ -229,4 +229,4 @@ RES_int %1, 64*(%2), %3 RES_int %1, _UNION_SIZE, _UNION_ALIGN %endm -%endif ; end ifdef _DATASTRUCT_ASM_ +%endif ; end ifdef _DATASTRUCT_INC_ diff --git a/lib/include/dbgprint.asm b/lib/include/dbgprint.inc similarity index 98% rename from lib/include/dbgprint.asm rename to lib/include/dbgprint.inc index 30fe3dc2bd88cdb970b89ee2261c56004f5054e9..1cd66928692902f1c8a3165bbde26d0c19382889 100644 --- a/lib/include/dbgprint.asm +++ b/lib/include/dbgprint.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -72,8 +72,8 @@ ; Printing the debug data also resets the write pointer to the beginning, ; effectively "deleting" the previous messages. ; -%ifndef DBGPRINT_ASM_INCLUDED -%define DBGPRINT_ASM_INCLUDED +%ifndef DBGPRINT_INC_INCLUDED +%define DBGPRINT_INC_INCLUDED ;%define DO_DBGPRINT %ifdef DO_DBGPRINT @@ -408,4 +408,4 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %endif ; ifdef 0 ; OLD -%endif ; DBGPRINT_ASM_INCLUDED +%endif ; DBGPRINT_INC_INCLUDED diff --git a/lib/include/des.h b/lib/include/des.h index f8944b3aff54d9a22ceac1d18c627c544273f10c..39ae99ea6a5a10f756eb94ee682ec776350cdab5 100644 --- a/lib/include/des.h +++ b/lib/include/des.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/des_utils.h b/lib/include/des_utils.h index 72f5ddcc345185a97e2ddc7922e4d5a4f4f4719b..d88b96218c4d4d944a948168dcc4fd2693963541 100644 --- a/lib/include/des_utils.h +++ b/lib/include/des_utils.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/docsis_common.h b/lib/include/docsis_common.h index ca57ab9539db7fe905fa182e435b8d2cea0979ad..daea83278b97b1b7635b3dbabd458e67d6713676 100644 --- a/lib/include/docsis_common.h +++ b/lib/include/docsis_common.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -58,27 +58,6 @@ #define IMB_AES_BLOCK_SIZE 16 #endif -IMB_DLL_LOCAL void aes_cfb_256_one_sse_no_aesni(void *out, const void *in, - const void *iv, - const void *keys, - const uint64_t len); - -IMB_DLL_LOCAL void aes_cfb_256_one_sse(void *out, const void *in, - const void *iv, const void *keys, - const uint64_t len); - -IMB_DLL_LOCAL void aes_cfb_256_one_avx(void *out, const void *in, - const void *iv, const void *keys, - const uint64_t len); - -IMB_DLL_LOCAL void aes_cfb_256_one_avx2(void *out, const void *in, - const void *iv, const void *keys, - const uint64_t len); - -IMB_DLL_LOCAL void aes_cfb_256_one_avx512(void *out, const void *in, - const void *iv, const void *keys, - const uint64_t len); - /** * @brief Encrypts/decrypts the last partial block for DOCSIS SEC v3.1 BPI * diff --git a/lib/include/error.h b/lib/include/error.h index 73d44f51806b97463aec3eab223c704d85370754..ec9936f088bf84c12b0be59784ae87d8762210ab 100644 --- a/lib/include/error.h +++ b/lib/include/error.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -32,7 +32,7 @@ * declare global variable to store * process wide error status */ -extern int imb_errno; +extern volatile int imb_errno; /** * @brief API to set error status @@ -47,8 +47,12 @@ void imb_set_errno(IMB_MGR *mb_mgr, const int errnum) if (mb_mgr != NULL) mb_mgr->imb_errno = errnum; - /* set global error status */ - imb_errno = errnum; + /* + * set global error status + * (only if different, to limit unneeded stores) + */ + if (imb_errno != errnum) + imb_errno = errnum; } #endif /* ERROR_H */ diff --git a/lib/include/error.inc b/lib/include/error.inc index 54df265f764d2cd6d5bb192920da1827134ea00a..28c397223283c9b492ef6d2d716f4ce86c1c7b23 100644 --- a/lib/include/error.inc +++ b/lib/include/error.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -28,7 +28,7 @@ %ifndef ERROR_ASM_INCLUDED %define ERROR_ASM_INCLUDED -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" extern imb_errno extern imb_errno_types @@ -98,12 +98,16 @@ SET_ERRNO_TYPES \ IMB_ERR_NULL_BURST, \ IMB_ERR_BURST_SIZE, \ IMB_ERR_BURST_OOO, \ - IMB_ERR_SELFTEST + IMB_ERR_SELFTEST, \ + IMB_ERR_BURST_SUITE_ID ;; Reset global imb_errno to 0 %macro IMB_ERR_CHECK_RESET 0 + cmp dword [rel imb_errno], 0 + jz %%do_not_reset mov dword [rel imb_errno], 0 +%%do_not_reset: %endmacro ;; Set GP reg and global imb_errno to 0 diff --git a/lib/include/gcm.h b/lib/include/gcm.h index 82ba93b481e4ba4254f2b4333c9a7d0a6274d5d3..b933da713eb2cc8d8a7ea600a90e1c841049c107 100644 --- a/lib/include/gcm.h +++ b/lib/include/gcm.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -517,7 +517,6 @@ aes_gcm_init_var_iv_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, const uint64_t iv_len, const uint8_t *aad, const uint64_t aad_len); - IMB_DLL_EXPORT void aes_gcm_init_var_iv_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -533,312 +532,76 @@ aes_gcm_init_var_iv_256_avx_gen4(const struct gcm_key_data *key_data, * Internal GCM API for SSE/AVX/AVX2/AVX512/AESNI emulation, * to be used only through job API. */ +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_128_sse(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_192_sse(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_256_sse(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_128_sse(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_192_sse(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_256_sse(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_128_avx_gen4(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_192_avx_gen4(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_256_avx_gen4(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_128_avx_gen4(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_192_avx_gen4(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_256_avx_gen4(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_128_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_192_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_256_avx512(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_128_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_192_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_256_avx512(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_128_vaes_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_192_vaes_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_256_vaes_avx512(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_128_vaes_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_192_vaes_avx512(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_256_vaes_avx512(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_128_sse_no_aesni(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_192_sse_no_aesni(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_enc_var_iv_256_sse_no_aesni(IMB_MGR *state, IMB_JOB *job); + +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_128_sse_no_aesni(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_192_sse_no_aesni(IMB_MGR *state, IMB_JOB *job); +IMB_DLL_LOCAL IMB_JOB * +aes_gcm_dec_var_iv_256_sse_no_aesni(IMB_MGR *state, IMB_JOB *job); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_128_sse(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_192_sse(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_256_sse(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_128_sse(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_192_sse(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_256_sse(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); - -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_128_avx_gen2(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_192_avx_gen2(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_256_avx_gen2(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_128_avx_gen2(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_192_avx_gen2(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_256_avx_gen2(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); - -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_128_avx_gen4(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_192_avx_gen4(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_256_avx_gen4(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_128_avx_gen4(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_192_avx_gen4(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_256_avx_gen4(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); - -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_128_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_192_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_256_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_128_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_192_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_256_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, const uint64_t auth_tag_len); - -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_128_vaes_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_192_vaes_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_256_vaes_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_128_vaes_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_192_vaes_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_256_vaes_avx512(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); - -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_128_sse_no_aesni(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_192_sse_no_aesni(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_enc_var_iv_256_sse_no_aesni(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_128_sse_no_aesni(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_192_sse_no_aesni(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); -IMB_DLL_LOCAL void -aes_gcm_dec_var_iv_256_sse_no_aesni(const struct gcm_key_data *key_data, - struct gcm_context_data *context_data, - uint8_t *out, const uint8_t *in, - const uint64_t msg_len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, const uint64_t aad_len, - uint8_t *auth_tag, - const uint64_t auth_tag_len); /* * GHASH API for SSE/AVX/AVX2/AVX512/AESNI emulation */ @@ -875,6 +638,12 @@ ghash_pre_sse(const void *key, struct gcm_key_data *key_data); IMB_DLL_EXPORT void ghash_pre_avx_gen2(const void *key, struct gcm_key_data *key_data); +IMB_DLL_EXPORT void +ghash_pre_avx_gen4(const void *key, struct gcm_key_data *key_data); + +IMB_DLL_EXPORT void +ghash_pre_avx512(const void *key, struct gcm_key_data *key_data); + IMB_DLL_EXPORT void ghash_pre_vaes_avx512(const void *key, struct gcm_key_data *key_data); diff --git a/lib/sse_t1/gcm_api_sse.inc b/lib/include/gcm_api_sse.inc similarity index 59% rename from lib/sse_t1/gcm_api_sse.inc rename to lib/include/gcm_api_sse.inc index 774286eb9cbcac88c82d16a705e43f350412523a..90d681ecf9256b62400fc01481ce0f3b2a630014 100644 --- a/lib/sse_t1/gcm_api_sse.inc +++ b/lib/include/gcm_api_sse.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -29,7 +29,8 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; -%include "sse_t1/gcm_sse.inc" +%include "include/gcm_sse.inc" +%include "include/imb_job.inc" %ifndef GCM_API_SSE_INC %define GCM_API_SSE_INC @@ -41,6 +42,11 @@ default rel ;void aes_gcm_precomp_128_sse / aes_gcm_precomp_192_sse / aes_gcm_precomp_256_sse ; (struct gcm_key_data *key_data); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(precomp,_),function,) +FN_NAME_AVX(precomp,_): +%endif MKGLOBAL(FN_NAME(precomp,_),function,) FN_NAME(precomp,_): endbranch64 @@ -53,19 +59,10 @@ FN_NAME(precomp,_): jz error_precomp %endif - push r12 - push r13 - push r14 - push r15 - - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 ; align rsp to 64 bytes - %ifidn __OUTPUT_FORMAT__, win64 + sub rsp, 1*16 ; only xmm6 needs to be maintained - movdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 + movdqu [rsp + 0*16], xmm6 %endif pxor xmm6, xmm6 @@ -95,14 +92,9 @@ FN_NAME(precomp,_): clear_scratch_xmms_sse_asm %endif %ifidn __OUTPUT_FORMAT__, win64 - movdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] + movdqu xmm6, [rsp + 0*16] + add rsp, 1*16 %endif - mov rsp, r14 - - pop r15 - pop r14 - pop r13 - pop r12 exit_precomp: @@ -135,6 +127,11 @@ error_precomp: ; u8 *auth_tag, ; u64 auth_tag_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(enc,_),function,) +FN_NAME_AVX(enc,_): +%endif MKGLOBAL(FN_NAME(enc,_),function,) FN_NAME(enc,_): endbranch64 @@ -270,6 +267,11 @@ skip_aad_check_error_enc: ; u8 *auth_tag, ; u64 auth_tag_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(dec,_),function,) +FN_NAME_AVX(dec,_): +%endif MKGLOBAL(FN_NAME(dec,_),function,) FN_NAME(dec,_): endbranch64 @@ -395,308 +397,108 @@ skip_aad_check_error_dec: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_var_iv_128_sse / aes_gcm_enc_var_iv_192_sse / -; aes_gcm_enc_var_iv_256_sse -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); +; +; NOTE: THIS API IS USED BY JOB-API ONLY, NO NEED FOR 2ND SAFE PARAM CHECK +; +;IMB_JOB *aes_gcm_enc_var_iv_128_sse / aes_gcm_enc_var_iv_192_sse / +; aes_gcm_enc_var_iv_256_sse(IMB_MGR *state, IMB_JOB *job) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc_var_iv,_),function,) +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(enc_var_iv,_),function,internal) +FN_NAME_AVX(enc_var_iv,_): +%endif +MKGLOBAL(FN_NAME(enc_var_iv,_),function,internal) FN_NAME(enc_var_iv,_): + endbranch64 + FUNC_SAVE alloc_context - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_enc_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_enc_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_enc_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_enc_IV - - cmp arg11, 16 - ja error_enc_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_enc_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_enc_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_enc_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_enc_IV - -skip_in_out_check_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_enc_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_enc_IV + mov arg1, [arg2 + _enc_keys] -skip_aad_check_enc_IV: -%endif - cmp arg7, 12 + cmp qword [arg2 + _iv_len_in_bytes], 12 je iv_len_12_enc_IV - GCM_INIT arg1, arg2, arg6, arg8, arg9, arg7 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]}, \ + {[arg2 + _iv_len_in_bytes]} + jmp skip_iv_len_12_enc_IV iv_len_12_enc_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]} skip_iv_len_12_enc_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC + mov arg3, [arg2 + _src] + add arg3, [arg2 + _cipher_start_src_offset] + mov arg4, [arg2 + _dst] + mov [rsp + GP_OFFSET + 5*8], arg2 ; preserve job pointer + mov arg2, [arg2 + _msg_len_to_cipher] + GCM_ENC_DEC arg1, {rsp + CONTEXT_OFFSET}, arg4, arg3, arg2, ENC - GCM_COMPLETE arg1, arg2, arg10, arg11 + mov arg2, [rsp + GP_OFFSET + 5*8] + GCM_COMPLETE arg1, {rsp + CONTEXT_OFFSET}, \ + {[arg2 + _auth_tag_output]}, {[arg2 + _auth_tag_output_len_in_bytes]} -exit_enc_IV: - FUNC_RESTORE - - ret + ;; mark job complete + mov dword [arg2 + _status], IMB_STATUS_COMPLETED -%ifdef SAFE_PARAM -error_enc_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV + mov rax, arg2 ;; return the job - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_enc_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_enc_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_enc_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc_IV -%endif + FUNC_RESTORE + ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_var_iv_128_sse / aes_gcm_dec_var_iv_192_sse / -; aes_gcm_dec_var_iv_256_sse -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); +; +; NOTE: THIS API IS USED BY JOB-API ONLY, NO NEED FOR 2ND SAFE PARAM CHECK +; +;IMB_JOB *aes_gcm_dec_var_iv_128_sse / aes_gcm_dec_var_iv_192_sse / +; aes_gcm_dec_var_iv_256_sse(IMB_MGR *state, IMB_JOB *job) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec_var_iv,_),function,) +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(dec_var_iv,_),function,internal) +FN_NAME_AVX(dec_var_iv,_): +%endif +MKGLOBAL(FN_NAME(dec_var_iv,_),function,internal) FN_NAME(dec_var_iv,_): + endbranch64 + FUNC_SAVE alloc_context - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_dec_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_dec_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_dec_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_dec_IV - - cmp arg11, 16 - ja error_dec_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_dec_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_dec_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_dec_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_dec_IV - -skip_in_out_check_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_dec_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_dec_IV + mov arg1, [arg2 + _dec_keys] -skip_aad_check_dec_IV: -%endif - cmp arg7, 12 + cmp qword [arg2 + _iv_len_in_bytes], 12 je iv_len_12_dec_IV - GCM_INIT arg1, arg2, arg6, arg8, arg9, arg7 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]}, \ + {[arg2 + _iv_len_in_bytes]} + jmp skip_iv_len_12_dec_IV iv_len_12_dec_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]} skip_iv_len_12_dec_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC - - GCM_COMPLETE arg1, arg2, arg10, arg11 - -exit_dec_IV: - FUNC_RESTORE - - ret - -%ifdef SAFE_PARAM -error_dec_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV + mov arg3, [arg2 + _src] + add arg3, [arg2 + _cipher_start_src_offset] + mov arg4, [arg2 + _dst] + mov [rsp + GP_OFFSET + 5*8], arg2 ; preserve job pointer + mov arg2, [arg2 + _msg_len_to_cipher] + GCM_ENC_DEC arg1, {rsp + CONTEXT_OFFSET}, arg4, arg3, arg2, DEC - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN + mov arg2, [rsp + GP_OFFSET + 5*8] + GCM_COMPLETE arg1, {rsp + CONTEXT_OFFSET}, \ + {[arg2 + _auth_tag_output]}, {[arg2 + _auth_tag_output_len_in_bytes]} - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_dec_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_dec_IV + ;; mark job complete + mov dword [arg2 + _status], IMB_STATUS_COMPLETED - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD + mov rax, arg2 ;; return the job -skip_aad_check_error_dec_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec_IV -%endif + FUNC_RESTORE + ret mksection stack-noexec %endif ; GCM_API_SSE_INC diff --git a/lib/avx512_t2/gcm_api_vaes_avx512.inc b/lib/include/gcm_api_vaes_avx512.inc similarity index 59% rename from lib/avx512_t2/gcm_api_vaes_avx512.inc rename to lib/include/gcm_api_vaes_avx512.inc index 1b76667d6473572fd9e646c40f76fed147bbf32e..a897c13c5f40fdff95de327eae4bc4d658c35953 100644 --- a/lib/avx512_t2/gcm_api_vaes_avx512.inc +++ b/lib/include/gcm_api_vaes_avx512.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,9 +27,11 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512_t2/gcm_vaes_avx512.inc" +%include "include/gcm_vaes_avx512.inc" %include "include/error.inc" -%include "include/os.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/imb_job.inc" %ifndef GCM_API_VAES_AVX512_INC %define GCM_API_VAES_AVX512_INC @@ -56,7 +58,7 @@ FN_NAME(precomp,_): jz error_precomp %endif - FUNC_SAVE + FUNC_SAVE small_frame vpxor xmm6, xmm6 ENCRYPT_SINGLE_BLOCK arg1, xmm6 ; xmm6 = HashKey @@ -76,10 +78,14 @@ FN_NAME(precomp,_): vpand xmm2, xmm2, [rel POLY] vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vmovdqu [arg1 + HashKey], xmm6 ; store HashKey<<1 mod poly + vmovdqu [arg1 + HashKey_1], xmm6 ; store HashKey<<1 mod poly PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm7, xmm8 +%ifdef SAFE_DATA + clear_zmms_avx512 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 +%endif + FUNC_RESTORE exit_precomp: @@ -179,6 +185,9 @@ skip_aad_check_enc: zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, single_call GCM_COMPLETE arg1, arg2, arg9, arg10, single_call, k1, r10, r11, r12 +%ifdef SAFE_DATA + clear_zmms_avx512 xmm6 +%endif exit_enc: FUNC_RESTORE @@ -313,7 +322,9 @@ skip_aad_check_dec: zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call GCM_COMPLETE arg1, arg2, arg9, arg10, single_call, k1, r10, r11, r12 - +%ifdef SAFE_DATA + clear_zmms_avx512 xmm6 +%endif exit_dec: FUNC_RESTORE ret @@ -369,313 +380,115 @@ skip_aad_check_error_dec: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_var_iv_128_vaes_avx512 / aes_gcm_enc_var_iv_192_vaes_avx512 / -; aes_gcm_enc_var_iv_256_vaes_avx512 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); +; +; NOTE: THIS API IS USED BY JOB-API ONLY, NO NEED FOR 2ND SAFE PARAM CHECK +; +;IMB_JOB *aes_gcm_enc_var_iv_128_vaes_avx512 / aes_gcm_enc_var_iv_192_vaes_avx512 / +; aes_gcm_enc_var_iv_256_vaes_avx512(IMB_MGR *state, IMB_JOB *job) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc_var_iv,_),function,) +MKGLOBAL(FN_NAME(enc_var_iv,_),function,internal) FN_NAME(enc_var_iv,_): endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_enc_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_enc_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_enc_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_enc_IV - - cmp arg11, 16 - ja error_enc_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_enc_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_enc_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_enc_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_enc_IV + FUNC_SAVE alloc_context -skip_in_out_check_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_enc_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_enc_IV + mov arg1, [arg2 + _enc_keys] -skip_aad_check_enc_IV: -%endif - cmp arg7, 12 + cmp qword [arg2 + _iv_len_in_bytes], 12 je iv_len_12_enc_IV - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12, k1, xmm14, xmm2, \ - zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ - zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call, arg7 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {qword [arg2 + _gcm_aad_len]}, \ + r10, r11, r12, k1, xmm14, xmm2, \ + zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ + zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call, \ + {[arg2 + _iv_len_in_bytes]} jmp skip_iv_len_12_enc_IV iv_len_12_enc_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12, k1, xmm14, xmm2, \ - zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ - zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {qword [arg2 + _gcm_aad_len]}, \ + r10, r11, r12, k1, xmm14, xmm2, \ + zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ + zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call skip_iv_len_12_enc_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, single_call - GCM_COMPLETE arg1, arg2, arg10, arg11, single_call, k1, r10, r11, r12 + mov arg3, [arg2 + _src] + add arg3, [arg2 + _cipher_start_src_offset] + mov arg4, [arg2 + _dst] + mov rbp, [arg2 + _msg_len_to_cipher] + GCM_ENC_DEC arg1, {rsp + CONTEXT_OFFSET}, arg4, arg3, rbp, ENC, single_call + + GCM_COMPLETE arg1, {rsp + CONTEXT_OFFSET}, \ + {[arg2 + _auth_tag_output]}, {[arg2 + _auth_tag_output_len_in_bytes]}, \ + single_call, k1, r10, r11, r12 + +%ifdef SAFE_DATA + clear_zmms_avx512 xmm1, xmm4, xmm6, xmm7, xmm8, xmm12, xmm13, xmm14, \ + xmm16, xmm17, xmm18, xmm19, xmm20, xmm21, xmm26, xmm30, xmm31 +%endif + ;; mark job complete + mov dword [arg2 + _status], IMB_STATUS_COMPLETED + + mov rax, arg2 ;; return the job -exit_enc_IV: FUNC_RESTORE ret -%ifdef SAFE_PARAM -error_enc_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_enc_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_enc_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_enc_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc_IV -%endif - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_var_iv_128_vaes_avx512 / aes_gcm_dec_var_iv_192_vaes_avx512 / -; aes_gcm_dec_var_iv_256_vaes_avx512 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); +; +; NOTE: THIS API IS USED BY JOB-API ONLY, NO NEED FOR 2ND SAFE PARAM CHECK +; +;IMB_JOB *aes_gcm_dec_var_iv_128_vaes_avx512 / aes_gcm_dec_var_iv_192_vaes_avx512 / +; aes_gcm_dec_var_iv_256_vaes_avx512(IMB_MGR *state, IMB_JOB *job) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec_var_iv,_),function,) +MKGLOBAL(FN_NAME(dec_var_iv,_),function,internal) FN_NAME(dec_var_iv,_): endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_dec_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_dec_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_dec_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_dec_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_dec_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_dec_IV - - cmp arg11, 16 - ja error_dec_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_dec_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_dec_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_dec_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_dec_IV - -skip_in_out_check_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_dec_IV + FUNC_SAVE alloc_context - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_dec_IV + mov arg1, [arg2 + _dec_keys] -skip_aad_check_dec_IV: -%endif - cmp arg7, 12 + cmp qword [arg2 + _iv_len_in_bytes], 12 je iv_len_12_dec_IV - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12, k1, xmm14, xmm2, \ - zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, zmm12, \ - zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call, arg7 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {qword [arg2 + _gcm_aad_len]}, \ + r10, r11, r12, k1, xmm14, xmm2, \ + zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ + zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call, \ + {qword [arg2 + _iv_len_in_bytes]} jmp skip_iv_len_12_dec_IV iv_len_12_dec_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12, k1, xmm14, xmm2, \ - zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ - zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {qword [arg2 + _gcm_aad_len]}, \ + r10, r11, r12, k1, xmm14, xmm2, \ + zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ + zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, single_call skip_iv_len_12_dec_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call - GCM_COMPLETE arg1, arg2, arg10, arg11, single_call, k1, r10, r11, r12 + mov arg3, [arg2 + _src] + add arg3, [arg2 + _cipher_start_src_offset] + mov arg4, [arg2 + _dst] + mov rbp, [arg2 + _msg_len_to_cipher] + GCM_ENC_DEC arg1, {rsp + CONTEXT_OFFSET}, arg4, arg3, rbp, DEC, single_call + + GCM_COMPLETE arg1, {rsp + CONTEXT_OFFSET}, \ + {[arg2 + _auth_tag_output]}, {[arg2 + _auth_tag_output_len_in_bytes]}, \ + single_call, k1, r10, r11, r12 + +%ifdef SAFE_DATA + clear_zmms_avx512 xmm1, xmm4, xmm6, xmm7, xmm8, xmm12, xmm13, xmm14, \ + xmm16, xmm17, xmm18, xmm19, xmm20, xmm21, xmm26, xmm30, xmm31 +%endif + ;; mark job complete + mov dword [arg2 + _status], IMB_STATUS_COMPLETED + + mov rax, arg2 ;; return the job -exit_dec_IV: FUNC_RESTORE ret -%ifdef SAFE_PARAM -error_dec_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax - - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_dec_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_dec_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD - -skip_aad_check_error_dec_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_dec_IV -%endif - mksection stack-noexec %endif ; GCM_API_VAES_AVX512_INC diff --git a/lib/include/gcm_avx512.inc b/lib/include/gcm_avx512.inc new file mode 100644 index 0000000000000000000000000000000000000000..2c97259683a25a07880bd006af5b005183d823b5 --- /dev/null +++ b/lib/include/gcm_avx512.inc @@ -0,0 +1,31 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define IS_AVX512_GCM +%include "include/gcm_common_avx2_avx512.inc" diff --git a/lib/avx_t1/aes128_gcm_by8_avx.asm b/lib/include/gcm_avx_gen4.inc similarity index 92% rename from lib/avx_t1/aes128_gcm_by8_avx.asm rename to lib/include/gcm_avx_gen4.inc index dba11298c7feee02dac5b2c997952549b5b83557..12f596c47c161cd90b9aaefbcee5f8bb6f4a03f6 100644 --- a/lib/avx_t1/aes128_gcm_by8_avx.asm +++ b/lib/include/gcm_avx_gen4.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2011-2022 Intel Corporation All rights reserved. +; Copyright(c) 2011-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,5 +27,5 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%define GCM128_MODE 1 -%include "avx_t1/gcm_avx_gen2.asm" +%define IS_AVX2_GCM +%include "include/gcm_common_avx2_avx512.inc" diff --git a/lib/include/gcm_common.inc b/lib/include/gcm_common.inc index 94d39783feb3a18075bc49936b5612b7617b4f6b..7cab9760aeedf0fa3825b9cefba30334806b99df 100644 --- a/lib/include/gcm_common.inc +++ b/lib/include/gcm_common.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -28,7 +28,7 @@ %ifndef _GCM_COMMON_INC_ %define _GCM_COMMON_INC_ -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" ;; *** ;; *** SSE and AVX @@ -38,69 +38,31 @@ ; READ_SMALL_DATA_INPUT: Packs xmm register with data when data input is less than 16 bytes. ; Returns 0 if data has length 0. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro READ_SMALL_DATA_INPUT 7 +%macro READ_SMALL_DATA_INPUT 5 %define %%OUTPUT %1 ; [out] xmm register %define %%INPUT %2 ; [in] gp with input address %define %%LENGTH %3 ; [in] gp with data length (0 to 16 bytes) -%define %%UNUSED %4 ; [unused] gp register -%define %%COUNTER %5 ; [clobbered] gp register -%define %%TMP1 %6 ; [clobbered] gp register -%define %%ARCH %7 ; [in] "SSE" or "AVX" +%define %%TMP1 %4 ; [clobbered] temporary gp register +%define %%ARCH %5 ; [in] "SSE" or "AVX" -%ifidn %%ARCH, SSE - pxor %%OUTPUT, %%OUTPUT -%else - vpxor %%OUTPUT, %%OUTPUT -%endif - mov %%COUNTER, %%LENGTH - or %%COUNTER, %%COUNTER - jz %%_done - - xor %%TMP1, %%TMP1 - - cmp %%COUNTER, 8 - jb %%_byte_loop_2 -%ifidn %%ARCH, SSE - movq %%OUTPUT, [%%INPUT] -%else - vmovq %%OUTPUT, [%%INPUT] -%endif - je %%_done - - sub %%COUNTER, 8 - -%%_byte_loop_1: ;Read 1 byte at a time - shl %%TMP1, 8 ;This loop handles case when 8 bytes were read in - dec %%COUNTER - mov BYTE(%%TMP1), BYTE [%%INPUT + %%COUNTER + 8] - jnz %%_byte_loop_1 -%ifidn %%ARCH, SSE - pinsrq %%OUTPUT, %%TMP1, 1 -%else - vpinsrq %%OUTPUT, %%TMP1, 1 -%endif - jmp %%_done + mov DWORD(%%TMP1), 16 + cmp %%LENGTH, %%TMP1 + cmovb %%TMP1, %%LENGTH -%%_byte_loop_2: ;Read 1 byte at a time - shl %%TMP1, 8 ;This loop handles case when no bytes were read in - dec %%COUNTER - mov BYTE(%%TMP1), BYTE [%%INPUT + %%COUNTER] - jnz %%_byte_loop_2 %ifidn %%ARCH, SSE - movq %%OUTPUT, %%TMP1 + simd_load_sse_16 %%OUTPUT, %%INPUT, %%TMP1 %else - vmovq %%OUTPUT, %%TMP1 + simd_load_avx_16 %%OUTPUT, %%INPUT, %%TMP1 %endif -%%_done: %endmacro ; READ_SMALL_DATA_INPUT -%macro READ_SMALL_DATA_INPUT_SSE 6 - READ_SMALL_DATA_INPUT %1, %2, %3, %4, %5, %6, SSE +%macro READ_SMALL_DATA_INPUT_SSE 4 + READ_SMALL_DATA_INPUT %1, %2, %3, %4, SSE %endmacro ; READ_SMALL_DATA_INPUT_SSE -%macro READ_SMALL_DATA_INPUT_AVX 6 - READ_SMALL_DATA_INPUT %1, %2, %3, %4, %5, %6, AVX +%macro READ_SMALL_DATA_INPUT_AVX 4 + READ_SMALL_DATA_INPUT %1, %2, %3, %4, AVX %endmacro ; READ_SMALL_DATA_INPUT_AVX ;; *** diff --git a/lib/avx512_t1/gcm_avx512.asm b/lib/include/gcm_common_avx2_avx512.inc similarity index 81% rename from lib/avx512_t1/gcm_avx512.asm rename to lib/include/gcm_common_avx2_avx512.inc index 352a89c5aa9177fa9cb69e80d3992940b16084a5..4d6d06c41ae29e0b677ce0dbc614e10c1baf96b0 100644 --- a/lib/avx512_t1/gcm_avx512.asm +++ b/lib/include/gcm_common_avx2_avx512.inc @@ -1,5 +1,5 @@ -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2011-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -111,66 +111,101 @@ ; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part. ; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" -%include "include/gcm_defines.asm" -%include "include/gcm_keys_avx2_avx512.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" +%include "include/gcm_defines.inc" +%include "include/gcm_keys_avx2_avx512.inc" %include "include/gcm_common.inc" +%include "include/memcpy.inc" %include "include/cet.inc" -%include "include/mb_mgr_datastruct.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" %include "include/error.inc" +%include "include/imb_job.inc" %ifndef GCM128_MODE %ifndef GCM192_MODE %ifndef GCM256_MODE -%error "No GCM mode selected for gcm_avx512.asm!" +%error "No GCM key size selected for gcm_avx2_avx512.inc!" +%endif +%endif +%endif + +%ifndef IS_AVX2_GCM +%ifndef IS_AVX512_GCM +%error "No GCM AVX2 or AVX512 selection made for gcm_avx2_avx512.inc!" +%endif %endif + +%ifdef IS_AVX2_GCM +%xdefine GCM_API_POSTFIX avx_gen4 %endif + +%ifdef IS_AVX512_GCM +%xdefine GCM_API_POSTFIX avx512 %endif ;; Decide on AES-GCM key size to compile for %ifdef GCM128_MODE %define NROUNDS 9 -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ avx512 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _128_ %+ avx512 +%xdefine GCM_API_KEYSZ _128 %endif %ifdef GCM192_MODE %define NROUNDS 11 -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ avx512 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _192_ %+ avx512 +%xdefine GCM_API_KEYSZ _192 %endif %ifdef GCM256_MODE %define NROUNDS 13 -%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ avx512 -%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _256_ %+ avx512 +%xdefine GCM_API_KEYSZ _256 %endif +;; Decide on AES-GCM key size to compile for +%define FN_NAME(x,y) aes_gcm_ %+ x %+ GCM_API_KEYSZ %+ y %+ GCM_API_POSTFIX +%define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ GCM_API_KEYSZ %+ _ %+ GCM_API_POSTFIX +%define GHASH_FN_NAME(x) x %+ _ %+ GCM_API_POSTFIX + mksection .text default rel -; need to push 4 registers into stack to maintain -%define STACK_OFFSET 8*4 +; need to store 5 GP registers on stack (align to 16 bytes) +%define GP_STORAGE 8*6 + +%define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register) +%define TMP3 16*1 ; Temporary storage for AES State 3 +%define TMP4 16*2 ; Temporary storage for AES State 4 +%define TMP5 16*3 ; Temporary storage for AES State 5 +%define TMP6 16*4 ; Temporary storage for AES State 6 +%define TMP7 16*5 ; Temporary storage for AES State 7 +%define TMP8 16*6 ; Temporary storage for AES State 8 + +%define LOCAL_STORAGE 16*7 %ifidn __OUTPUT_FORMAT__, win64 - %define XMM_STORAGE 16*10 + %define XMM_STORAGE 16*10 %else - %define XMM_STORAGE 0 + %define XMM_STORAGE 0 %endif -%define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register) -%define TMP3 16*1 ; Temporary storage for AES State 3 -%define TMP4 16*2 ; Temporary storage for AES State 4 -%define TMP5 16*3 ; Temporary storage for AES State 5 -%define TMP6 16*4 ; Temporary storage for AES State 6 -%define TMP7 16*5 ; Temporary storage for AES State 7 -%define TMP8 16*6 ; Temporary storage for AES State 8 -%define LOCAL_STORAGE 16*7 -%define VARIABLE_OFFSET LOCAL_STORAGE + XMM_STORAGE +%define GP_OFFSET (LOCAL_STORAGE + XMM_STORAGE) + +%define VARIABLE_OFFSET (GP_STORAGE + LOCAL_STORAGE + XMM_STORAGE) + +;; extra memory for GCM context structure +%define CONTEXT_SIZE 6*16 +%define CONTEXT_OFFSET VARIABLE_OFFSET + +;; Full stack frame layout: +;; RETURN ADDRESS + ARGS +;; R14 = + 16*6 -> --------------------------- +;; GCM CONTEXT (JOB API only) +;; + 6*8 -> --------------------------- +;; GP STORAGE +;; + 16*10 -> -------------------------- +;; XMM STORAGE (windows only) +;; + 16*7 -> -------------------------- +;; LOCAL STORAGE +;; RSP = -> -------------------------- ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; Utility Macros @@ -207,7 +242,7 @@ default rel ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;first phase of the reduction - vmovdqu %%T3, [rel POLY2] + vmovdqa %%T3, [rel POLY2] vpclmulqdq %%T2, %%T3, %%GH, 0x01 vpslldq %%T2, %%T2, 8 ; shift-L %%T2 2 DWs @@ -224,22 +259,19 @@ default rel vpxor %%GH, %%GH, %%T2 ; second phase of the reduction complete ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; vpxor %%GH, %%GH, %%T1 ; the result is in %%GH + %endmacro -; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx512 -; functions, but are kept to allow users to switch cpu architectures between calls -; of pre, init, update, and finalize. %macro PRECOMPUTE 8 -%define %%GDATA %1 -%define %%HK %2 -%define %%T1 %3 -%define %%T2 %4 -%define %%T3 %5 -%define %%T4 %6 -%define %%T5 %7 -%define %%T6 %8 - - ; Haskey_i_k holds XORed values of the low and high parts of the Haskey_i +%define %%GDATA %1 ;; [in/out] pointer to GCM key data structure +%define %%HK %2 ;; [in] Hash Key +%define %%T1 %3 ;; [clobbered] temporary XMM register +%define %%T2 %4 ;; [clobbered] temporary XMM register +%define %%T3 %5 ;; [clobbered] temporary XMM register +%define %%T4 %6 ;; [clobbered] temporary XMM register +%define %%T5 %7 ;; [clobbered] temporary XMM register +%define %%T6 %8 ;; [clobbered] temporary XMM register + vmovdqa %%T5, %%HK GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^2<<1 mod poly @@ -269,20 +301,34 @@ default rel ; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). ; Output: The hash of the data (AAD_HASH). ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro CALC_AAD_HASH 13 -%define %%A_IN %1 -%define %%A_LEN %2 -%define %%AAD_HASH %3 -%define %%GDATA_KEY %4 -%define %%XTMP0 %5 ; xmm temp reg 5 -%define %%XTMP1 %6 ; xmm temp reg 5 -%define %%XTMP2 %7 -%define %%XTMP3 %8 -%define %%XTMP4 %9 -%define %%XTMP5 %10 ; xmm temp reg 5 -%define %%T1 %11 ; temp reg 1 -%define %%T2 %12 -%define %%T3 %13 +%macro CALC_AAD_HASH 13-15 +%define %%A_IN %1 ;; [in] message pointer +%define %%A_LEN %2 ;; [in] message length +%define %%AAD_HASH %3 ;; [in] input hash value (XMM) +%define %%GDATA_KEY %4 ;; [in] pointer to GCM key data +%define %%XTMP0 %5 ;; [clobbered] temporary XMM +%define %%XTMP1 %6 ;; [clobbered] temporary XMM +%define %%XTMP2 %7 ;; [clobbered] temporary XMM +%define %%XTMP3 %8 ;; [clobbered] temporary XMM +%define %%XTMP4 %9 ;; [clobbered] temporary XMM +%define %%XTMP5 %10 ;; [clobbered] temporary XMM +%define %%T1 %11 ;; [clobbered] temporary GP register +%define %%T2 %12 ;; [clobbered] temporary GP register +%define %%T3 %13 ;; [clobbered] temporary GP register +%define %%T4 %14 ;; [clobbered] temporary GP register (obsolete with avx512) +%define %%T5 %15 ;; [clobbered] temporary GP register (obsolete with avx512) + +%ifdef IS_AVX2_GCM +%if %0 != 15 +%error "AVX2 CALC_AAD_HASH needs 15 arguments!" +%endif +%endif + +%ifdef IS_AVX512_GCM +%if %0 != 13 +%error "AVX512 CALC_AAD_HASH needs 13 arguments!" +%endif +%endif mov %%T1, %%A_IN ; T1 = AAD mov %%T2, %%A_LEN ; T2 = aadLen @@ -436,7 +482,11 @@ default rel je %%_CALC_AAD_done vmovdqu %%XTMP0, [%%GDATA_KEY + HashKey] - READ_SMALL_DATA_INPUT_AVX512 %%XTMP1, %%T1, %%T2, %%T3, k1 +%ifdef IS_AVX2_GCM + READ_SMALL_DATA_INPUT_AVX %%XTMP1, %%T1, %%T2, %%T3 +%else + READ_SMALL_DATA_INPUT_AVX512 %%XTMP1, %%T1, %%T2, %%T3, k1 +%endif ;byte-reflect the AAD data vpshufb %%XTMP1, [rel SHUF_MASK] vpxor %%AAD_HASH, %%XTMP1 @@ -449,7 +499,7 @@ default rel ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks between update calls. ; Requires the input data be at least 1 byte long. -; Input: gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), +; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), ; input text length (PLAIN_CYPH_LEN), the current data offset (DATA_OFFSET), ; the hash subkey (HASH_SUBKEY) and whether encoding or decoding (ENC_DEC) ; Output: A cypher of the first partial block (CYPH_PLAIN_OUT), and updated GDATA_CTX @@ -466,12 +516,24 @@ default rel %define %%ENC_DEC %8 mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 + or r13, r13 je %%_partial_block_done ;Leave Macro if no partial blocks +%ifdef IS_AVX2_GCM + cmp %%PLAIN_CYPH_LEN, 16 ;Read in input data without over reading + jl %%_fewer_than_16_bytes + VXLDR xmm1, [%%PLAIN_CYPH_IN] ;If more than 16 bytes of data, just fill the xmm register + jmp %%_data_read + +%%_fewer_than_16_bytes: + lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + READ_SMALL_DATA_INPUT_AVX xmm1, r10, %%PLAIN_CYPH_LEN, rax + +%%_data_read: ;Finished reading in data +%else ; Read in input data without over reading READ_SMALL_DATA_INPUT_LEN_BT16_AVX512 xmm1, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, r12, rax, k1 - +%endif vmovdqu xmm9, [%%GDATA_CTX + PBlockEncKey] ;xmm9 = my_ctx_data.partial_block_enc_key lea r12, [rel SHIFT_MASK] @@ -505,13 +567,14 @@ default rel vpshufb xmm9, xmm2 vpxor %%AAD_HASH, xmm9 %endif - cmp r15,0 + or r15, r15 jl %%_partial_incomplete GHASH_MUL %%AAD_HASH, %%HASH_SUBKEY, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block - xor rax,rax + xor rax, rax mov [%%GDATA_CTX + PBlockLen], rax jmp %%_enc_dec_done + %%_partial_incomplete: %ifidn __OUTPUT_FORMAT__, win64 mov rax, %%PLAIN_CYPH_LEN @@ -529,7 +592,7 @@ default rel ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; output encrypted Bytes - cmp r15,0 + or r15, r15 jl %%_partial_fill mov r12, r13 mov r13, 16 @@ -538,10 +601,18 @@ default rel %%_partial_fill: mov r13, %%PLAIN_CYPH_LEN %%_count_set: + +%ifdef IS_AVX2_GCM + simd_store_avx %%CYPH_PLAIN_OUT, xmm9, r13, rax, r12, %%DATA_OFFSET + add %%DATA_OFFSET, r13 +%else lea rax, [rel byte_len_to_mask_table] kmovw k1, [rax + r13*2] vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET]{k1}, xmm9 add %%DATA_OFFSET, r13 +%endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + %%_partial_block_done: %endmacro ; PARTIAL_BLOCK @@ -619,34 +690,33 @@ default rel %assign i (9-%%num_initial_blocks) %rep %%num_initial_blocks - vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 - vmovdqa reg(i), %%CTR - vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap + vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 + vpshufb reg(i), %%CTR, [rel SHUF_MASK] ; perform a 16Byte swap %assign i (i+1) %endrep -%if(%%num_initial_blocks>0) -vmovdqu %%T_key, [%%GDATA_KEY+16*0] -%assign i (9-%%num_initial_blocks) +%if (%%num_initial_blocks > 0) + vmovdqu %%T_key, [%%GDATA_KEY+16*0] +%assign i (9 - %%num_initial_blocks) %rep %%num_initial_blocks - vpxor reg(i),reg(i),%%T_key + vpxor reg(i), reg(i), %%T_key %assign i (i+1) %endrep %assign j 1 %rep NROUNDS -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) + vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9 - %%num_initial_blocks) %rep %%num_initial_blocks - vaesenc reg(i),%%T_key + vaesenc reg(i), %%T_key %assign i (i+1) %endrep %assign j (j+1) %endrep -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) + vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9 - %%num_initial_blocks) %rep %%num_initial_blocks vaesenclast reg(i),%%T_key %assign i (i+1) @@ -654,35 +724,58 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %endif ; %if(%%num_initial_blocks>0) -%assign i (9-%%num_initial_blocks) +%assign i (9 - %%num_initial_blocks) +%assign k 0 +%rep %%num_initial_blocks + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + k*16] + vpxor reg(i), reg(i), %%T1 + +%ifidn %%ENC_DEC, DEC +%if k == 0 + vmovdqa %%T6, %%T1 +%else + vmovdqa [rsp + TMP %+ i], %%T1 +%endif +%endif + +%assign i (i + 1) +%assign k (k + 1) +%endrep + +%assign i (9 - %%num_initial_blocks) +%assign k 0 %rep %%num_initial_blocks - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - vpxor reg(i), reg(i), %%T1 ;; Write back ciphertext for %%num_initial_blocks blocks - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) - add %%DATA_OFFSET, 16 - %ifidn %%ENC_DEC, DEC - vmovdqa reg(i), %%T1 - %endif + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + k*16], reg(i) ;; Prepare ciphertext for GHASH computations - vpshufb reg(i), [rel SHUF_MASK] -%assign i (i+1) +%ifidn %%ENC_DEC, DEC +%if k == 0 + vpshufb reg(i), %%T6, [rel SHUF_MASK] +%else + vmovdqa %%T1, [rsp + TMP %+ i] + vpshufb reg(i), %%T1, [rel SHUF_MASK] +%endif +%else + vpshufb reg(i), reg(i), [rel SHUF_MASK] +%endif +%assign i (i + 1) +%assign k (k + 1) %endrep + add %%DATA_OFFSET, (16 * %%num_initial_blocks) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%assign i (9-%%num_initial_blocks) -%if(%%num_initial_blocks>0) +%assign i (9 - %%num_initial_blocks) +%if (%%num_initial_blocks > 0) vmovdqa %%T3, reg(i) -%assign i (i+1) +%assign i (i + 1) %endif -%if %%num_initial_blocks>1 -%rep %%num_initial_blocks-1 +%if (%%num_initial_blocks > 1) +%rep %%num_initial_blocks - 1 vmovdqu [rsp + TMP %+ i], reg(i) %assign i (i+1) %endrep %endif - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Prepare 8 counter blocks and perform rounds of AES cipher on ;; them, load plain/cipher text and store cipher/plain text. @@ -996,7 +1089,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; First phase of the reduction - vmovdqu %%T3, [rel POLY2] + vmovdqa %%T3, [rel POLY2] vpclmulqdq %%T2, %%T3, %%T4, 0x01 vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs @@ -1020,65 +1113,15 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] vpxor %%T3, %%T1, %%T4 %else ;; The hash should end up in T3 - vmovdqa %%T3, %%T2 + vmovdqa %%T3, %%T2 %endif ;; Final hash is now in T3 -%if %%num_initial_blocks > 0 - ;; NOTE: obsolete in case %%num_initial_blocks = 0 - sub %%LENGTH, 16*%%num_initial_blocks -%endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*0] - vpxor %%XMM1, %%XMM1, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*0], %%XMM1 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM1, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*1] - vpxor %%XMM2, %%XMM2, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*1], %%XMM2 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM2, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*2] - vpxor %%XMM3, %%XMM3, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*2], %%XMM3 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM3, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*3] - vpxor %%XMM4, %%XMM4, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*3], %%XMM4 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM4, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*4] - vpxor %%XMM5, %%XMM5, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*4], %%XMM5 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM5, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*5] - vpxor %%XMM6, %%XMM6, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*5], %%XMM6 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM6, %%T1 - %endif - - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*6] - vpxor %%XMM7, %%XMM7, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*6], %%XMM7 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM7, %%T1 - %endif %if %%num_initial_blocks > 0 + ;; NOTE: obsolete in case %%num_initial_blocks = 0 + sub %%LENGTH, 16 * %%num_initial_blocks + ;; NOTE: 'jl' is never taken for %%num_initial_blocks = 0 ;; This macro is executed for length 128 and up, ;; zero length is checked in GCM_ENC_DEC. @@ -1088,30 +1131,97 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] cmp %%LENGTH, 128 jl %%_initial_skip_last_word_write %endif - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*7] - vpxor %%XMM8, %%XMM8, %%T1 - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*7], %%XMM8 - %ifidn %%ENC_DEC, DEC - vmovdqa %%XMM8, %%T1 - %endif + + ;; Load 8 plain/cipher text blocks and XOR them against AES blocks +%assign i 1 +%rep 8 +%assign k (i - 1) + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*k] + vpxor reg(i), reg(i), %%T1 +%ifidn %%ENC_DEC, DEC +%if i == 1 + vmovdqa %%T6, %%T1 +%else + vmovdqa [rsp + TMP %+ i], %%T1 +%endif +%endif +%assign i (i + 1) +%endrep + + ;; Store 8 cipher/plain text blocks and prepare cipher text blocks for GHASH +%assign i 1 +%rep 8 +%assign k (i - 1) + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*k], reg(i) +%ifidn %%ENC_DEC, DEC +%if i == 1 + vmovdqa reg(i), %%T6 +%else + vmovdqa reg(i), [rsp + TMP %+ i] +%endif +%endif +%assign i (i + 1) +%endrep ;; Update %%LENGTH with the number of blocks processed - sub %%LENGTH, 16 - add %%DATA_OFFSET, 16 + sub %%LENGTH, 8*16 + add %%DATA_OFFSET, 8*16 + +%if %%num_initial_blocks > 0 + ;; jmp and %%_initial_skip_last_word_write not required for %%num_initial_blocks=0 case + jmp %%_initial_words_done + %%_initial_skip_last_word_write: - sub %%LENGTH, 128-16 - add %%DATA_OFFSET, 128-16 + ;; Load 7 plain/cipher text blocks and XOR them against AES blocks +%assign i 1 +%rep 7 +%assign k (i - 1) + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*k] + vpxor reg(i), reg(i), %%T1 +%ifidn %%ENC_DEC, DEC +%if i == 1 + vmovdqa %%T6, %%T1 +%else + vmovdqa [rsp + TMP %+ i], %%T1 +%endif +%endif +%assign i (i + 1) +%endrep + + ;; Store 7 cipher/plain text blocks and prepare cipher text blocks for GHASH +%assign i 1 +%rep 7 +%assign k (i - 1) + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*k], reg(i) +%ifidn %%ENC_DEC, DEC +%if i == 1 + vmovdqa reg(i), %%T6 +%else + vmovdqa reg(i), [rsp + TMP %+ i] +%endif +%endif +%assign i (i + 1) +%endrep + + ;; Update %%LENGTH with the number of blocks processed + sub %%LENGTH, 7*16 + add %%DATA_OFFSET, 7*16 + +%%_initial_words_done: +%endif ;; %%num_initial_blocks > 0 - vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap ;; Combine GHASHed value with the corresponding ciphertext - vpxor %%XMM1, %%XMM1, %%T3 - vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + vpxor %%XMM1, %%XMM1, %%T3 + vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1123,16 +1233,16 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;;; num_initial_blocks is expected to include the partial final block ;;; in the count. %macro INITIAL_BLOCKS_PARTIAL 25 -%define %%GDATA_KEY %1 -%define %%GDATA_CTX %2 -%define %%CYPH_PLAIN_OUT %3 -%define %%PLAIN_CYPH_IN %4 -%define %%LENGTH %5 -%define %%DATA_OFFSET %6 -%define %%num_initial_blocks %7 ; can be 1, 2, 3, 4, 5, 6 or 7 (not 0) +%define %%GDATA_KEY %1 ;; [in] pointer to GCM key data +%define %%GDATA_CTX %2 ;; [in] pointer to GCM context data +%define %%CYPH_PLAIN_OUT %3 ;; [in] pointer to destination buffer +%define %%PLAIN_CYPH_IN %4 ;; [in] pointer to source buffer +%define %%LENGTH %5 ;; [in] message length +%define %%DATA_OFFSET %6 ;; [in/out] buffer offset +%define %%num_initial_blocks %7 ;; [in] numeric value, number of blocks can be from 1 to 7 (not 0) %define %%T1 %8 %define %%T2 %9 -%define %%T3 %10 ; [out] hash value +%define %%T3 %10 ;; [out] hash value (XMM) %define %%T4 %11 %define %%T5 %12 %define %%CTR %13 @@ -1143,70 +1253,94 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %define %%XMM5 %18 %define %%XMM6 %19 %define %%XMM7 %20 -%define %%XMM8 %21 ; [in] hash value +%define %%XMM8 %21 ;; [in] hash value (XMM) %define %%T6 %22 %define %%T_key %23 %define %%ENC_DEC %24 %define %%INSTANCE_TYPE %25 +%assign i (8 - %%num_initial_blocks) ;; Move AAD_HASH to temp reg vmovdqu %%T2, %%XMM8 + ;; vmovdqu %%CTR, [%%GDATA_CTX + CurCount] ; %%CTR = Y0 -%assign i (9-%%num_initial_blocks) +%assign i (9 - %%num_initial_blocks) %rep %%num_initial_blocks ;; Compute AES counters vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 vmovdqa reg(i), %%CTR vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap -%assign i (i+1) +%assign i (i + 1) %endrep -vmovdqu %%T_key, [%%GDATA_KEY+16*0] -%assign i (9-%%num_initial_blocks) + vmovdqu %%T_key, [%%GDATA_KEY+16*0] +%assign i (9 - %%num_initial_blocks) %rep %%num_initial_blocks ; Start AES for %%num_initial_blocks blocks - vpxor reg(i),reg(i),%%T_key -%assign i (i+1) + vpxor reg(i), reg(i), %%T_key +%assign i (i + 1) %endrep %assign j 1 %rep NROUNDS -vmovdqu %%T_key, [%%GDATA_KEY+16*j] + vmovdqu %%T_key, [%%GDATA_KEY+16*j] %assign i (9-%%num_initial_blocks) %rep %%num_initial_blocks - vaesenc reg(i),%%T_key -%assign i (i+1) + vaesenc reg(i), %%T_key +%assign i (i + 1) %endrep -%assign j (j+1) +%assign j (j + 1) %endrep -vmovdqu %%T_key, [%%GDATA_KEY+16*j] -%assign i (9-%%num_initial_blocks) + vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9 - %%num_initial_blocks) %rep %%num_initial_blocks vaesenclast reg(i),%%T_key -%assign i (i+1) +%assign i (i + 1) %endrep ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Hash all but the last block of data ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%assign i (9-%%num_initial_blocks) -%rep %%num_initial_blocks-1 +%assign k 0 +%assign i (9 - %%num_initial_blocks) +%rep %%num_initial_blocks - 1 ;; Encrypt the message for all but the last block - VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - vpxor reg(i), reg(i), %%T1 - ;; write back ciphertext for %%num_initial_blocks blocks - VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) - add %%DATA_OFFSET, 16 + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*k] + vpxor reg(i), reg(i), %%T1 %ifidn %%ENC_DEC, DEC - vmovdqa reg(i), %%T1 +%if k == 0 + vmovdqa %%T6, %%T1 +%else + vmovdqa [rsp + TMP %+ i], %%T1 +%endif %endif +%assign i (i + 1) +%assign k (k + 1) +%endrep + +%assign k 0 +%assign i (9 - %%num_initial_blocks) +%rep %%num_initial_blocks - 1 + ;; write back ciphertext for %%num_initial_blocks blocks + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*k], reg(i) ;; Prepare ciphertext for GHASH computations - vpshufb reg(i), [rel SHUF_MASK] -%assign i (i+1) +%ifidn %%ENC_DEC, DEC +%if k == 0 + vpshufb reg(i), %%T6, [rel SHUF_MASK] +%else + vmovdqa reg(i), [rsp + TMP %+ i] + vpshufb reg(i), reg(i), [rel SHUF_MASK] +%endif +%else ; ENC + vpshufb reg(i), reg(i), [rel SHUF_MASK] +%endif +%assign i (i + 1) +%assign k (k + 1) %endrep + add %%DATA_OFFSET, (16 * (%%num_initial_blocks - 1)) %if %%num_initial_blocks > 1 ;; The final block of data may be <16B @@ -1232,19 +1366,20 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; write back ciphertext for %%num_initial_blocks blocks VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) add %%DATA_OFFSET, 16 + ;; Prepare ciphertext for GHASH computations %ifidn %%ENC_DEC, DEC - vmovdqa reg(i), %%T1 + vpshufb reg(i), %%T1, [rel SHUF_MASK] +%else + vpshufb reg(i), reg(i), [rel SHUF_MASK] %endif - ;; Prepare ciphertext for GHASH computations - vpshufb reg(i), [rel SHUF_MASK] ;; Hash all of the data -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) +%assign i (8 - %%num_initial_blocks) +%assign j (9 - %%num_initial_blocks) %assign k (%%num_initial_blocks) %assign last_block_to_hash 0 -%if(%%num_initial_blocks>last_block_to_hash) +%if (%%num_initial_blocks > last_block_to_hash) ;; Hash in AES state vpxor %%T2, reg(j) @@ -1286,7 +1421,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %endrep ;; Record that a reduction is needed - mov r12, 1 + mov r12, 1 jmp %%_small_initial_compute_hash @@ -1312,22 +1447,22 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; NOTE: could be replaced with %%LENGTH but at this point ;; %%LENGTH is always less than 16. ;; No PLAIN_CYPH_LEN argument available in this macro. - ENCRYPT_FINAL_PARTIAL_BLOCK reg(i), %%T1, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, LT16, %%ENC_DEC, %%DATA_OFFSET + ENCRYPT_FINAL_PARTIAL_BLOCK reg(i), %%T1, %%T3, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, LT16, %%ENC_DEC, %%DATA_OFFSET vpshufb reg(i), [rel SHUF_MASK] %ifidn %%INSTANCE_TYPE, multi_call -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) -%assign k (%%num_initial_blocks-1) +%assign i (8 - %%num_initial_blocks) +%assign j (9 - %%num_initial_blocks) +%assign k (%%num_initial_blocks - 1) %assign last_block_to_hash 1 %else -%assign i (8-%%num_initial_blocks) -%assign j (9-%%num_initial_blocks) +%assign i (8 - %%num_initial_blocks) +%assign j (9 - %%num_initial_blocks) %assign k (%%num_initial_blocks) %assign last_block_to_hash 0 %endif -%if(%%num_initial_blocks>last_block_to_hash) +%if (%%num_initial_blocks > last_block_to_hash) ;; Record that a reduction is needed mov r12, 1 ;; Hash in AES state @@ -1348,7 +1483,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; Record that a reduction is not needed - ;; In this case no hashes are computed because there ;; is only one initial block and it is < 16B in length. - xor r12, r12 + xor r12, r12 %endif %assign i (i+1) @@ -1398,7 +1533,6 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; initial_blocks == 1 and init/update/final is being used. ;; In this case we may just have a partial block, and that ;; gets hashed in finalize. - ;; cmp r12, 0 or r12, r12 je %%_no_reduction_needed %endif @@ -1411,7 +1545,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; First phase of the reduction - vmovdqu %%T3, [rel POLY2] + vmovdqa %%T3, [rel POLY2] vpclmulqdq %%T2, %%T3, %%T4, 0x01 ;; shift-L xmm2 2 DWs @@ -1512,7 +1646,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %ifidn %%loop_idx, in_order vpaddd %%XMM1, %%CTR, [rel ONE] ; INCR CNT - vmovdqu %%T5, [rel TWO] + vmovdqa %%T5, [rel TWO] vpaddd %%XMM2, %%CTR, %%T5 vpaddd %%XMM3, %%XMM1, %%T5 vpaddd %%XMM4, %%XMM2, %%T5 @@ -1522,7 +1656,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] vpaddd %%XMM8, %%XMM6, %%T5 vmovdqa %%CTR, %%XMM8 - vmovdqu %%T5, [rel SHUF_MASK] + vmovdqa %%T5, [rel SHUF_MASK] vpshufb %%XMM1, %%T5 ; perform a 16Byte swap vpshufb %%XMM2, %%T5 ; perform a 16Byte swap vpshufb %%XMM3, %%T5 ; perform a 16Byte swap @@ -1533,7 +1667,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] vpshufb %%XMM8, %%T5 ; perform a 16Byte swap %else vpaddd %%XMM1, %%CTR, [rel ONEf] ; INCR CNT - vmovdqu %%T5, [rel TWOf] + vmovdqa %%T5, [rel TWOf] vpaddd %%XMM2, %%CTR, %%T5 vpaddd %%XMM3, %%XMM1, %%T5 vpaddd %%XMM4, %%XMM2, %%T5 @@ -1805,76 +1939,48 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] vmovdqu %%T5, [%%GDATA + 16*14] %endif ; GCM256 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; accumulate product into %%T1:%%T7 + + vpslldq %%T3, %%T6, 8 ; shift-L %%T3 2 DWs + vpsrldq %%T6, %%T6, 8 ; shift-R %%T2 2 DWs + vpxor %%T7, %%T7, %%T3 + vpxor %%T1, %%T1, %%T6 ; accumulate the results in %%T1:%%T7 + %assign i 0 %assign j 1 %rep 8 - ;; SNP TBD: This is pretty ugly - consider whether just XORing the - ;; data in after vaesenclast is simpler and performant. Would - ;; also have to ripple it through partial block and ghash_mul_8. -%ifidn %%FULL_PARTIAL, full - %ifdef NT_LD - VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - vpxor %%T2, %%T2, %%T5 - %else - vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - %endif - - %ifidn %%ENC_DEC, ENC - vaesenclast reg(j), reg(j), %%T2 - %else - vaesenclast %%T3, reg(j), %%T2 - vpxor reg(j), %%T2, %%T5 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 - %endif +%assign skip_load_and_xor 0 +%ifnidn %%FULL_PARTIAL, full +%if i >= 7 +;; if partial case and dealing with the last block then skip load and xor +%assign skip_load_and_xor 1 +%endif +%endif -%else - ; Don't read the final data during partial block processing - %ifdef NT_LD - %if (i<7) - VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - vpxor %%T2, %%T2, %%T5 - %else - ;; Stage the key directly in T2 rather than hash it with plaintext - vmovdqu %%T2, %%T5 - %endif - %else - %if (i<7) - vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - %else - ;; Stage the key directly in T2 rather than hash it with plaintext - vmovdqu %%T2, %%T5 - %endif - %endif - - %ifidn %%ENC_DEC, ENC - vaesenclast reg(j), reg(j), %%T2 - %else - %if (i<7) - vaesenclast %%T3, reg(j), %%T2 - vpxor reg(j), %%T2, %%T5 - ;; Do not read the data since it could fault - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 - %else - vaesenclast reg(j), reg(j), %%T2 - %endif - %endif +%if skip_load_and_xor == 0 + VXLDR %%T2, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*i] %endif + vaesenclast reg(j), reg(j), %%T5 +%if skip_load_and_xor == 0 + vpxor reg(j), reg(j), %%T2 +%ifidn %%ENC_DEC, DEC +%if i == 0 + vmovdqa %%T6, %%T2 +%else + vmovdqa [rsp + TMP %+ j], %%T2 +%endif ; i +%endif ; DEC +%endif ; skip_load_and_xor -%assign i (i+1) -%assign j (j+1) +%assign i (i + 1) +%assign j (j + 1) %endrep -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - vpslldq %%T3, %%T6, 8 ; shift-L %%T3 2 DWs - vpsrldq %%T6, %%T6, 8 ; shift-R %%T2 2 DWs - vpxor %%T7, %%T7, %%T3 - vpxor %%T1, %%T1, %%T6 ; accumulate the results in %%T1:%%T7 - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;first phase of the reduction - vmovdqu %%T3, [rel POLY2] + ;; first phase of the reduction + vmovdqa %%T3, [rel POLY2] vpclmulqdq %%T2, %%T3, %%T7, 0x01 vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs @@ -1882,23 +1988,34 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - %ifidn %%ENC_DEC, ENC - ; Write to the Ciphertext buffer - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*0], %%XMM1 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*1], %%XMM2 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*2], %%XMM3 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*3], %%XMM4 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*4], %%XMM5 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*5], %%XMM6 - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*6], %%XMM7 - %ifidn %%FULL_PARTIAL, full - ;; Avoid writing past the buffer if handling a partial block - VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*7], %%XMM8 - %endif - %endif - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;second phase of the reduction +%assign i 0 +%assign j 1 +%rep 8 +%assign skip_store 0 +%ifnidn %%FULL_PARTIAL, full +%if i >= 7 +;; if partial case and dealing with the last block then skip store +%assign skip_store 1 +%endif +%endif + +%if skip_store == 0 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*i], reg(j) +%ifidn %%ENC_DEC, DEC +%if i == 0 + vmovdqa reg(j), %%T6 +%else + vmovdqa reg(j), [rsp + TMP %+ j] +%endif ; i +%endif ; DEC +%endif ; skip_store + +%assign i (i + 1) +%assign j (j + 1) +%endrep + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; second phase of the reduction vpclmulqdq %%T2, %%T3, %%T7, 0x00 vpsrldq %%T2, %%T2, 4 ; shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) @@ -1916,7 +2033,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap - vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap vpxor %%XMM1, %%T1 @@ -2091,7 +2208,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;first phase of the reduction - vmovdqu %%T3, [rel POLY2] + vmovdqa %%T3, [rel POLY2] vpclmulqdq %%T2, %%T3, %%T7, 0x01 vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs @@ -2264,7 +2381,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;first phase of the reduction - vmovdqu %%T3, [rel POLY2] + vmovdqa %%T3, [rel POLY2] vpclmulqdq %%T2, %%T3, %%T7, 0x01 vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs @@ -2289,24 +2406,98 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;;; r13 - Number of bytes to read ;;; MODIFIES: ;;; KEY - Key for encrypting the partial block +;;; HASH - Current hash value ;;; SMASHES: -;;; rax, T1 -;;; Note: -;;; PLAIN_CYPH_LEN is unused at this stage. Previously: -;;; it was used to determine if buffer is big enough to do -;;; a 16 byte read & shift. +;;; r10, r12, r15, rax +;;; T1, T2 +;;; Note AVX2: +;;; PLAIN_CYPH_LEN, %7, is passed only to determine +;;; if buffer is big enough to do a 16 byte read & shift. ;;; 'LT16' is passed here only if buffer is known to be smaller ;;; than 16 bytes. ;;; Any other value passed here will result in 16 byte read ;;; code path. -%macro ENCRYPT_FINAL_PARTIAL_BLOCK 7 +;;; Note AVX512: +;;; PLAIN_CYPH_LEN and T2 are unused at this stage. +%macro ENCRYPT_FINAL_PARTIAL_BLOCK 8 %define %%KEY %1 %define %%T1 %2 -%define %%CYPH_PLAIN_OUT %3 -%define %%PLAIN_CYPH_IN %4 -%define %%PLAIN_CYPH_LEN %5 -%define %%ENC_DEC %6 -%define %%DATA_OFFSET %7 +%define %%T2 %3 +%define %%CYPH_PLAIN_OUT %4 +%define %%PLAIN_CYPH_IN %5 +%define %%PLAIN_CYPH_LEN %6 +%define %%ENC_DEC %7 +%define %%DATA_OFFSET %8 + +%ifdef IS_AVX2_GCM + + ;; NOTE: type of read tuned based %%PLAIN_CYPH_LEN setting +%ifidn %%PLAIN_CYPH_LEN, LT16 + ;; Handle the case where the message is < 16 bytes + lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + + ;; T1 - packed output + ;; r10 - input data address + ;; r13 - input data length + ;; r12, r15, rax - temp registers + READ_SMALL_DATA_INPUT_AVX %%T1, r10, r13, r12 + + lea r12, [SHIFT_MASK + 16] + sub r12, r13 +%else + ;; Handle the case where the message is >= 16 bytes + sub %%DATA_OFFSET, 16 + add %%DATA_OFFSET, r13 + ;; Receive the last <16 Byte block + vmovdqu %%T1, [%%PLAIN_CYPH_IN+%%DATA_OFFSET] + sub %%DATA_OFFSET, r13 + add %%DATA_OFFSET, 16 + + lea r12, [SHIFT_MASK + 16] + ;; Adjust the shuffle mask pointer to be able to shift 16-r13 bytes + ;; (r13 is the number of bytes in plaintext mod 16) + sub r12, r13 + ;; Get the appropriate shuffle mask + vmovdqu %%T2, [r12] + ;; shift right 16-r13 bytes + vpshufb %%T1, %%T2 +%endif ; %%PLAIN_CYPH_LEN, LT16 + + ;; At this point T1 contains the partial block data +%ifidn %%ENC_DEC, DEC + ;; Plaintext XOR E(K, Yn) + ;; Set aside the ciphertext + vmovdqa %%T2, %%T1 + vpxor %%KEY, %%KEY, %%T1 + ;; Get the appropriate mask to mask out top 16-r13 bytes of ciphertext + vmovdqu %%T1, [r12 + ALL_F - SHIFT_MASK] + ;; Mask out top 16-r13 bytes of ciphertext + vpand %%KEY, %%KEY, %%T1 + + ;; Prepare the ciphertext for the hash + ;; mask out top 16-r13 bytes of the plaintext + vpand %%T2, %%T2, %%T1 +%else + ;; Plaintext XOR E(K, Yn) + vpxor %%KEY, %%KEY, %%T1 + ;; Get the appropriate mask to mask out top 16-r13 bytes of %%KEY + vmovdqu %%T1, [r12 + ALL_F - SHIFT_MASK] + ;; Mask out top 16-r13 bytes of %%KEY + vpand %%KEY, %%KEY, %%T1 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Output r13 Bytes + vmovdqa %%T1, %%KEY + simd_store_avx %%CYPH_PLAIN_OUT, %%T1, r13, rax, r12, %%DATA_OFFSET + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%ifidn %%ENC_DEC, DEC + ;; If decrypt, restore the ciphertext into %%KEY + vmovdqa %%KEY, %%T2 +%endif + +%else ;; IS_AVX512_GCM ;; %%PLAIN_CYPH_IN + %%DATA_OFFSET ;; - input data address @@ -2331,6 +2522,9 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %else vmovdqu8 %%KEY{k1}{z}, %%KEY %endif + +%endif ;; IS_AVX512_GCM + %endmacro ; ENCRYPT_FINAL_PARTIAL_BLOCK ; Encryption of a single block @@ -2349,38 +2543,45 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; Start of Stack Setup -%macro FUNC_SAVE 0 - ;; Required for Update/GMC_ENC - ;the number of pushes must equal STACK_OFFSET - push r12 - push r13 - push r14 - push r15 - mov r14, rsp +%macro FUNC_SAVE 0-1 + ;; Required for Update/GCM_ENC + mov rax, rsp + +%if %0 == 0 + sub rsp, VARIABLE_OFFSET +%else +%ifidni %1, alloc_context + sub rsp, VARIABLE_OFFSET + CONTEXT_SIZE +%endif +%endif + and rsp, ~63 - sub rsp, VARIABLE_OFFSET - and rsp, ~63 + mov [rsp + GP_OFFSET + 0*8], rax ; original rsp pointer + mov [rsp + GP_OFFSET + 1*8], r12 + mov [rsp + GP_OFFSET + 2*8], r13 + mov [rsp + GP_OFFSET + 3*8], r14 + mov [rsp + GP_OFFSET + 4*8], r15 + mov r14, rax %ifidn __OUTPUT_FORMAT__, win64 ; xmm6:xmm15 need to be maintained for Windows - vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 - vmovdqu [rsp + LOCAL_STORAGE + 1*16],xmm7 - vmovdqu [rsp + LOCAL_STORAGE + 2*16],xmm8 - vmovdqu [rsp + LOCAL_STORAGE + 3*16],xmm9 - vmovdqu [rsp + LOCAL_STORAGE + 4*16],xmm10 - vmovdqu [rsp + LOCAL_STORAGE + 5*16],xmm11 - vmovdqu [rsp + LOCAL_STORAGE + 6*16],xmm12 - vmovdqu [rsp + LOCAL_STORAGE + 7*16],xmm13 - vmovdqu [rsp + LOCAL_STORAGE + 8*16],xmm14 - vmovdqu [rsp + LOCAL_STORAGE + 9*16],xmm15 + vmovdqu [rsp + LOCAL_STORAGE + 0*16], xmm6 + vmovdqu [rsp + LOCAL_STORAGE + 1*16], xmm7 + vmovdqu [rsp + LOCAL_STORAGE + 2*16], xmm8 + vmovdqu [rsp + LOCAL_STORAGE + 3*16], xmm9 + vmovdqu [rsp + LOCAL_STORAGE + 4*16], xmm10 + vmovdqu [rsp + LOCAL_STORAGE + 5*16], xmm11 + vmovdqu [rsp + LOCAL_STORAGE + 6*16], xmm12 + vmovdqu [rsp + LOCAL_STORAGE + 7*16], xmm13 + vmovdqu [rsp + LOCAL_STORAGE + 8*16], xmm14 + vmovdqu [rsp + LOCAL_STORAGE + 9*16], xmm15 %endif %endmacro %macro FUNC_RESTORE 0 %ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_zmms_asm + clear_scratch_xmms_avx_asm %endif %ifidn __OUTPUT_FORMAT__, win64 vmovdqu xmm15, [rsp + LOCAL_STORAGE + 9*16] @@ -2389,20 +2590,21 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] vmovdqu xmm12, [rsp + LOCAL_STORAGE + 6*16] vmovdqu xmm11, [rsp + LOCAL_STORAGE + 5*16] vmovdqu xmm10, [rsp + LOCAL_STORAGE + 4*16] - vmovdqu xmm9, [rsp + LOCAL_STORAGE + 3*16] - vmovdqu xmm8, [rsp + LOCAL_STORAGE + 2*16] - vmovdqu xmm7, [rsp + LOCAL_STORAGE + 1*16] - vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] + vmovdqu xmm9, [rsp + LOCAL_STORAGE + 3*16] + vmovdqu xmm8, [rsp + LOCAL_STORAGE + 2*16] + vmovdqu xmm7, [rsp + LOCAL_STORAGE + 1*16] + vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] %endif -;; Required for Update/GMC_ENC - mov rsp, r14 - pop r15 - pop r14 - pop r13 - pop r12 + + ;; Required for Update/GCM_ENC + mov r12, [rsp + GP_OFFSET + 1*8] + mov r13, [rsp + GP_OFFSET + 2*8] + mov r14, [rsp + GP_OFFSET + 3*8] + mov r15, [rsp + GP_OFFSET + 4*8] + mov rsp, [rsp + GP_OFFSET + 0*8] %endmacro -%macro CALC_J0 13 +%macro CALC_J0 15 %define %%KEY %1 ;; [in] Pointer to GCM KEY structure %define %%IV %2 ;; [in] Pointer to IV %define %%IV_LEN %3 ;; [in] IV length @@ -2410,20 +2612,27 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %define %%TMP0 %5 ;; [clobbered] Temporary GP reg %define %%TMP1 %6 ;; [clobbered] Temporary GP reg %define %%TMP2 %7 ;; [clobbered] Temporary GP reg -%define %%XTMP0 %8 ;; [clobbered] Temporary XMM reg -%define %%XTMP1 %9 ;; [clobbered] Temporary XMM reg -%define %%XTMP2 %10 ;; [clobbered] Temporary XMM reg -%define %%XTMP3 %11 ;; [clobbered] Temporary XMM reg -%define %%XTMP4 %12 ;; [clobbered] Temporary XMM reg -%define %%XTMP5 %13 ;; [clobbered] Temporary XMM reg +%define %%TMP3 %8 ;; [clobbered] Temporary GP reg (unused with AVX512) +%define %%TMP4 %9 ;; [clobbered] Temporary GP reg (unused with AVX512) +%define %%XTMP0 %10 ;; [clobbered] Temporary XMM reg +%define %%XTMP1 %11 ;; [clobbered] Temporary XMM reg +%define %%XTMP2 %12 ;; [clobbered] Temporary XMM reg +%define %%XTMP3 %13 ;; [clobbered] Temporary XMM reg +%define %%XTMP4 %14 ;; [clobbered] Temporary XMM reg +%define %%XTMP5 %15 ;; [clobbered] Temporary XMM reg ;; J0 = GHASH(IV || 0s+64 || len(IV)64) ;; s = 16 * RoundUp(len(IV)/16) - len(IV) */ ;; Calculate GHASH of (IV || 0s) vpxor %%J0, %%J0 +%ifdef IS_AVX2_GCM + CALC_AAD_HASH %%IV, %%IV_LEN, %%J0, %%KEY, %%XTMP0, %%XTMP1, %%XTMP2, \ + %%XTMP3, %%XTMP4, %%XTMP5, %%TMP0, %%TMP1, %%TMP2, %%TMP3, %%TMP4 +%else CALC_AAD_HASH %%IV, %%IV_LEN, %%J0, %%KEY, %%XTMP0, %%XTMP1, %%XTMP2, \ %%XTMP3, %%XTMP4, %%XTMP5, %%TMP0, %%TMP1, %%TMP2 +%endif ;; Calculate GHASH of last 16-byte block (0 || len(IV)64) vmovdqu %%XTMP0, [%%KEY + HashKey] @@ -2438,10 +2647,10 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; GCM_INIT initializes a gcm_context_data struct to prepare for encoding/decoding. -; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), IV, IV_LEN +; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), IV, IV_LEN, ; Additional Authentication data (A_IN), Additional Data length (A_LEN). -; Output: Updated GDATA_CTX with the hash of A_IN (AadHash) and initialized other parts of GDATA_CTX. -; Clobbers rax, r10-r13, and xmm0-xmm6 +; Output: Updated GDATA_CTX with the hash of A_IN (AadHash) and initialized other parts of GDATA. +; Clobbers rax, r10-r13 and xmm0-xmm6 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %macro GCM_INIT 8-9 %define %%GDATA_KEY %1 ; [in] GCM expanded keys pointer @@ -2457,8 +2666,11 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %define %%AAD_HASH xmm14 vpxor %%AAD_HASH, %%AAD_HASH - CALC_AAD_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, %%GPR1, %%GPR2, %%GPR3 - +%ifdef IS_AVX2_GCM + CALC_AAD_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, %%GPR1, %%GPR2, %%GPR3, r13, rax +%else + CALC_AAD_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, %%GPR1, %%GPR2, %%GPR3 +%endif mov %%GPR1, %%A_LEN vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH ; ctx_data.aad hash = aad_hash mov [%%GDATA_CTX + AadLen], %%GPR1 ; ctx_data.aad_length = aad_length @@ -2468,16 +2680,14 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] mov [%%GDATA_CTX + PBlockLen], %%GPR1 ; ctx_data.partial_block_length = 0 %if %0 == 9 ;; IV is different than 12 bytes - CALC_J0 %%GDATA_KEY, %%IV, %%IV_LEN, xmm2, r10, r11, r12, xmm0, xmm1, \ + CALC_J0 %%GDATA_KEY, %%IV, %%IV_LEN, xmm2, r10, r11, r12, r13, rax, xmm0, xmm1, \ xmm3, xmm4, xmm5, xmm6 %else ;; IV is 12 bytes ;; read 12 IV bytes and pad with 0x00000001 mov %%GPR2, %%IV - vmovd xmm3, [%%GPR2 + 8] - vpslldq xmm3, 8 - vmovq xmm2, [%%GPR2] - vmovdqa xmm4, [rel ONEf] - vpternlogq xmm2, xmm3, xmm4, 0xfe ; xmm2 = xmm2 or xmm3 or xmm4 + vmovdqa xmm2, [rel ONEf] ; read 12 IV bytes and pad with 0x00000001 + vpinsrq xmm2, [%%GPR2], 0 + vpinsrd xmm2, [%%GPR2 + 8], 2 %endif vmovdqu [%%GDATA_CTX + OrigIV], xmm2 ; ctx_data.orig_IV = iv @@ -2503,41 +2713,21 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; NOTE: the check below is obsolete in current implementation. The check is already done in GCM_ENC_DEC. ;; cmp %%NUM_BLOCKS, 0 ;; je %%_small_initial_blocks_encrypted - cmp %%NUM_BLOCKS, 8 - je %%_small_initial_num_blocks_is_8 cmp %%NUM_BLOCKS, 7 je %%_small_initial_num_blocks_is_7 - cmp %%NUM_BLOCKS, 6 - je %%_small_initial_num_blocks_is_6 + ja %%_small_initial_num_blocks_is_8 cmp %%NUM_BLOCKS, 5 je %%_small_initial_num_blocks_is_5 - cmp %%NUM_BLOCKS, 4 - je %%_small_initial_num_blocks_is_4 + ja %%_small_initial_num_blocks_is_6 cmp %%NUM_BLOCKS, 3 je %%_small_initial_num_blocks_is_3 + ja %%_small_initial_num_blocks_is_4 cmp %%NUM_BLOCKS, 2 je %%_small_initial_num_blocks_is_2 jmp %%_small_initial_num_blocks_is_1 %%_small_initial_num_blocks_is_8: - ;; r13 - %%LENGTH - ;; xmm12 - T1 - ;; xmm13 - T2 - ;; xmm14 - T3 - AAD HASH OUT when not producing 8 AES keys - ;; xmm15 - T4 - ;; xmm11 - T5 - ;; xmm9 - CTR - ;; xmm1 - XMM1 - Cipher + Hash when producing 8 AES keys - ;; xmm2 - XMM2 - ;; xmm3 - XMM3 - ;; xmm4 - XMM4 - ;; xmm5 - XMM5 - ;; xmm6 - XMM6 - ;; xmm7 - XMM7 - ;; xmm8 - XMM8 - AAD HASH IN - ;; xmm10 - T6 - ;; xmm0 - T_key INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 8, \ xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ @@ -2601,6 +2791,8 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE %%_small_initial_blocks_encrypted: + ;; Note: zero initial blocks not allowed. + %endmacro ; GCM_ENC_DEC_SMALL ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2669,7 +2861,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] sub r13, %%DATA_OFFSET ;; There may be no more data if it was consumed in the partial block. - cmp r13, 0 + or r13, r13 je %%_enc_dec_done %endif ; %%INSTANCE_TYPE, multi_call mov r10, r13 @@ -2697,19 +2889,15 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] and r12, 0x7 ; Still, don't allow 8 INITIAL blocks since this will ; can be handled by the x8 partial loop. - cmp r12, 0 je %%_initial_num_blocks_is_0 - cmp r12, 7 - je %%_initial_num_blocks_is_7 cmp r12, 6 + ja %%_initial_num_blocks_is_7 je %%_initial_num_blocks_is_6 - cmp r12, 5 - je %%_initial_num_blocks_is_5 cmp r12, 4 + ja %%_initial_num_blocks_is_5 je %%_initial_num_blocks_is_4 - cmp r12, 3 - je %%_initial_num_blocks_is_3 cmp r12, 2 + ja %%_initial_num_blocks_is_3 je %%_initial_num_blocks_is_2 jmp %%_initial_num_blocks_is_1 @@ -2764,7 +2952,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %%_initial_blocks_encrypted: ;; The entire message was encrypted processed in initial and now need to be hashed - cmp r13, 0 + or r13, r13 je %%_encrypt_done ;; Encrypt the final <16 byte (partial) block, then hash @@ -2827,7 +3015,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %%_encrypt_by_8_parallel_done: ;; Test to see if we need a by 8 with partial block. At this point ;; bytes remaining should be either zero or between 113-127. - cmp r13, 0 + or r13, r13 je %%_encrypt_done %%_encrypt_by_8_partial: @@ -2849,7 +3037,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; xmm8 - Final encrypted counter - need to hash with partial or full block ciphertext ;; GDATA, KEY, T1, T2 - ENCRYPT_FINAL_PARTIAL_BLOCK xmm8, xmm0, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET + ENCRYPT_FINAL_PARTIAL_BLOCK xmm8, xmm0, xmm10, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET vpshufb xmm8, [rel SHUF_MASK] @@ -2864,7 +3052,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; GDATA, T1, T2, T3, T4, T5, T6, T7, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 %ifidn %%INSTANCE_TYPE, multi_call mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 + or r13, r13 jz %%_hash_last_8 GHASH_LAST_7 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 ;; XOR the partial word into the hash @@ -2880,7 +3068,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %%_enc_dec_done: -%endmacro ; GCM_ENC_DEC +%endmacro ; GCM_ENC_DEC ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; GCM_COMPLETE Finishes Encryption/Decryption of last partial block after GCM_UPDATE finishes. @@ -2910,7 +3098,7 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] ;; Encrypt the final partial block. If we did this as a single call then ;; the partial block was handled in the main GCM_ENC_DEC macro. mov r12, [%%GDATA_CTX + PBlockLen] - cmp r12, 0 + or r12, r12 je %%_partial_done @@ -2954,14 +3142,14 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] simd_store_avx r10, xmm9, r11, r12, rax jmp %%_return_T_done %%_T_8: - vmovq rax, xmm9 + vmovq rax, xmm9 mov [r10], rax jmp %%_return_T_done %%_T_12: - vmovq rax, xmm9 + vmovq rax, xmm9 mov [r10], rax vpsrldq xmm9, xmm9, 8 - vmovd eax, xmm9 + vmovd eax, xmm9 mov [r10 + 8], eax jmp %%_return_T_done %%_T_16: @@ -2978,7 +3166,10 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] %endmacro ; GCM_COMPLETE ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_precomp_128_avx512 / +;void aes_gcm_precomp_128_avx_gen4 / +; aes_gcm_precomp_192_avx_gen4 / +; aes_gcm_precomp_256_avx_gen4 / +; aes_gcm_precomp_128_avx512 / ; aes_gcm_precomp_192_avx512 / ; aes_gcm_precomp_256_avx512 ; (struct gcm_key_data *key_data) @@ -2986,30 +3177,19 @@ vmovdqu %%T_key, [%%GDATA_KEY+16*j] MKGLOBAL(FN_NAME(precomp,_),function,) FN_NAME(precomp,_): endbranch64 -;; Parameter is passed through register %ifdef SAFE_PARAM ;; Reset imb_errno IMB_ERR_CHECK_RESET ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_precomp - %endif - push r12 - push r13 - push r14 - push r15 - - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 ; align rsp to 64 bytes - %ifidn __OUTPUT_FORMAT__, win64 + sub rsp, 1*16 ; only xmm6 needs to be maintained - vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 + vmovdqu [rsp + 0*16],xmm6 %endif vpxor xmm6, xmm6 @@ -3035,20 +3215,16 @@ FN_NAME(precomp,_): PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 %ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_zmms_asm + clear_scratch_xmms_avx_asm %endif + %ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] + vmovdqu xmm6, [rsp + 0*16] + add rsp, 1*16 %endif - mov rsp, r14 - - pop r15 - pop r14 - pop r13 - pop r12 exit_precomp: + ret %ifdef SAFE_PARAM @@ -3066,7 +3242,8 @@ error_precomp: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_init_128_avx512 / aes_gcm_init_192_avx512 / aes_gcm_init_256_avx512 +;void aes_gcm_init_128_avx_gen4 / aes_gcm_init_192_avx_gen4 / aes_gcm_init_256_avx_gen4 / +; aes_gcm_init_128_avx512 / aes_gcm_init_192_avx512 / aes_gcm_init_256_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, ; u8 *iv, @@ -3081,10 +3258,10 @@ FN_NAME(init,_): %ifidn __OUTPUT_FORMAT__, win64 push r14 push r15 - mov r14, rsp + lea r14, [rsp + 4*8] ; xmm6 needs to be maintained for Windows sub rsp, 1*16 - vmovdqu [rsp + 0*16], xmm6 + vmovdqu [rsp + 0*16], xmm6 %endif %ifdef SAFE_PARAM @@ -3092,15 +3269,15 @@ FN_NAME(init,_): IMB_ERR_CHECK_RESET ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_init ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_init ;; Check IV != NULL - cmp arg3, 0 + or arg3, arg3 jz error_init ;; Check if aad_len == 0 @@ -3108,7 +3285,7 @@ FN_NAME(init,_): jz skip_aad_check_init ;; Check aad != NULL (aad_len != 0) - cmp arg4, 0 + or arg4, arg4 jz error_init skip_aad_check_init: @@ -3116,13 +3293,13 @@ skip_aad_check_init: GCM_INIT arg1, arg2, arg3, arg4, arg5, r10, r11, r12 %ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_zmms_asm + clear_scratch_xmms_avx_asm %endif exit_init: + %ifidn __OUTPUT_FORMAT__, win64 vmovdqu xmm6 , [rsp + 0*16] - mov rsp, r14 + add rsp, 1*16 pop r15 pop r14 %endif @@ -3159,7 +3336,9 @@ skip_aad_check_error_init: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_init_var_iv_128_avx512 / aes_gcm_init_var_iv_192_avx512 / +;void aes_gcm_init_var_iv_128_avx_gen4 / aes_gcm_init_var_iv_192_avx_gen4 / +; aes_gcm_init_var_iv_256_avx_gen4 +; aes_gcm_init_var_iv_128_avx512 / aes_gcm_init_var_iv_192_avx512 / ; aes_gcm_init_var_iv_256_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, @@ -3171,15 +3350,16 @@ skip_aad_check_error_init: MKGLOBAL(FN_NAME(init_var_iv,_),function,) FN_NAME(init_var_iv,_): endbranch64 - push r12 - push r13 + push r12 + push r13 %ifidn __OUTPUT_FORMAT__, win64 push r14 push r15 - mov r14, rsp - ; xmm6 needs to be maintained for Windows - sub rsp, 1*16 + lea r14, [rsp + 4*8] + ; xmm6 & xmm14 need to be maintained for Windows + sub rsp, 2*16 vmovdqu [rsp + 0*16], xmm6 + vmovdqu [rsp + 1*16], xmm14 %endif %ifdef SAFE_PARAM @@ -3187,19 +3367,19 @@ FN_NAME(init_var_iv,_): IMB_ERR_CHECK_RESET ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_init_IV ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_init_IV ;; Check IV != NULL - cmp arg3, 0 + or arg3, arg3 jz error_init_IV ;; Check iv_len != 0 - cmp arg4, 0 + or arg4, arg4 jz error_init_IV ;; Check if aad_len == 0 @@ -3223,18 +3403,19 @@ iv_len_12_init_IV: skip_iv_len_12_init_IV: %ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_zmms_asm + clear_scratch_xmms_avx_asm %endif exit_init_IV: + %ifidn __OUTPUT_FORMAT__, win64 - vmovdqu xmm6 , [rsp + 0*16] - mov rsp, r14 + vmovdqu xmm6, [rsp + 0*16] + vmovdqu xmm14, [rsp + 1*16] + add rsp, 2*16 pop r15 pop r14 %endif - pop r13 - pop r12 + pop r13 + pop r12 ret %ifdef SAFE_PARAM @@ -3268,8 +3449,11 @@ skip_aad_check_error_init_IV: jmp exit_init_IV %endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_update_avx512 / aes_gcm_enc_192_update_avx512 / +;void aes_gcm_enc_128_update_avx_gen4 / aes_gcm_enc_192_update_avx_gen4 / +; aes_gcm_enc_128_update_avx_gen4 / +; aes_gcm_enc_128_update_avx512 / aes_gcm_enc_192_update_avx512 / ; aes_gcm_enc_256_update_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, @@ -3283,18 +3467,18 @@ FN_NAME(enc,_update_): FUNC_SAVE %ifdef SAFE_PARAM - ;; Reset imb_errno + ;; Reset imb_errno IMB_ERR_CHECK_RESET - ;; Load max len to reg on windows + ;; Load max len to reg on windows INIT_GCM_MAX_LENGTH ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_update_enc ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_update_enc ;; Check if msg_len == 0 @@ -3306,11 +3490,11 @@ FN_NAME(enc,_update_): ja error_update_enc ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 + or arg3, arg3 jz error_update_enc ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 + or arg4, arg4 jz error_update_enc %endif GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, multi_call @@ -3341,7 +3525,7 @@ error_update_enc: ;; Check out != NULL IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - ;; Check in != NULL (plaintext_len != 0) + ;; Check in != NULL (msg_len != 0) IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC skip_in_out_check_error_update_enc: @@ -3351,7 +3535,9 @@ skip_in_out_check_error_update_enc: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_update_avx512 / aes_gcm_dec_192_update_avx512 / +;void aes_gcm_dec_128_update_avx_gen4 / aes_gcm_dec_192_update_avx_gen4 / +; aes_gcm_dec_256_update_avx_gen4 / +; aes_gcm_dec_128_update_avx512 / aes_gcm_dec_192_update_avx512 / ; aes_gcm_dec_256_update_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, @@ -3372,11 +3558,11 @@ FN_NAME(dec,_update_): INIT_GCM_MAX_LENGTH ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_update_dec ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_update_dec ;; Check if msg_len == 0 @@ -3388,11 +3574,11 @@ FN_NAME(dec,_update_): ja error_update_dec ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 + or arg3, arg3 jz error_update_dec ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 + or arg4, arg4 jz error_update_dec %endif @@ -3400,6 +3586,7 @@ FN_NAME(dec,_update_): exit_update_dec: FUNC_RESTORE + ret %ifdef SAFE_PARAM @@ -3432,9 +3619,10 @@ skip_in_out_check_error_update_dec: jmp exit_update_dec %endif - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_finalize_avx512 / aes_gcm_enc_192_finalize_avx512 / +;void aes_gcm_enc_128_finalize_avx_gen4 / aes_gcm_enc_192_finalize_avx_gen4 / +; aes_gcm_enc_256_finalize_avx_gen4 / +; aes_gcm_enc_128_finalize_avx512 / aes_gcm_enc_192_finalize_avx512 / ; aes_gcm_enc_256_finalize_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, @@ -3444,31 +3632,29 @@ skip_in_out_check_error_update_dec: MKGLOBAL(FN_NAME(enc,_finalize_),function,) FN_NAME(enc,_finalize_): endbranch64 -;; All parameters are passed through registers %ifdef SAFE_PARAM ;; Reset imb_errno IMB_ERR_CHECK_RESET ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_enc_fin ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_enc_fin ;; Check auth_tag != NULL - cmp arg3, 0 + or arg3, arg3 jz error_enc_fin ;; Check auth_tag_len == 0 or > 16 - cmp arg4, 0 + or arg4, arg4 jz error_enc_fin cmp arg4, 16 ja error_enc_fin %endif - push r12 %ifidn __OUTPUT_FORMAT__, win64 @@ -3485,9 +3671,9 @@ FN_NAME(enc,_finalize_): GCM_COMPLETE arg1, arg2, arg3, arg4, multi_call %ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_zmms_asm + clear_scratch_xmms_avx_asm %endif + %ifidn __OUTPUT_FORMAT__, win64 vmovdqu xmm15, [rsp + 6*16] vmovdqu xmm14, [rsp + 5*16] @@ -3498,11 +3684,9 @@ FN_NAME(enc,_finalize_): vmovdqu xmm6, [rsp + 0*16] add rsp, 7*16 %endif - pop r12 - exit_enc_fin: - ret + ret %ifdef SAFE_PARAM error_enc_fin: @@ -3529,7 +3713,9 @@ error_enc_fin: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_finalize_avx512 / aes_gcm_dec_192_finalize_avx512 +;void aes_gcm_dec_128_finalize_avx_gen4 / aes_gcm_dec_192_finalize_avx_gen4 +; aes_gcm_dec_256_finalize_avx_gen4 / +; aes_gcm_dec_128_finalize_avx512 / aes_gcm_dec_192_finalize_avx512 ; aes_gcm_dec_256_finalize_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, @@ -3539,25 +3725,24 @@ error_enc_fin: MKGLOBAL(FN_NAME(dec,_finalize_),function,) FN_NAME(dec,_finalize_): endbranch64 -;; All parameters are passed through registers %ifdef SAFE_PARAM ;; Reset imb_errno IMB_ERR_CHECK_RESET ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_dec_fin ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_dec_fin ;; Check auth_tag != NULL - cmp arg3, 0 + or arg3, arg3 jz error_dec_fin ;; Check auth_tag_len == 0 or > 16 - cmp arg4, 0 + or arg4, arg4 jz error_dec_fin cmp arg4, 16 @@ -3580,8 +3765,7 @@ FN_NAME(dec,_finalize_): GCM_COMPLETE arg1, arg2, arg3, arg4, multi_call %ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_zmms_asm + clear_scratch_xmms_avx_asm %endif %ifidn __OUTPUT_FORMAT__, win64 vmovdqu xmm15, [rsp + 6*16] @@ -3595,8 +3779,8 @@ FN_NAME(dec,_finalize_): %endif pop r12 -exit_dec_fin: +exit_dec_fin: ret %ifdef SAFE_PARAM @@ -3624,7 +3808,8 @@ error_dec_fin: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_128_avx512 / aes_gcm_enc_192_avx512 / aes_gcm_enc_256_avx512 +;void aes_gcm_enc_128_avx_gen4 / aes_gcm_enc_192_avx_gen4 / aes_gcm_enc_256_avx_gen4 / +; aes_gcm_enc_128_avx512 / aes_gcm_enc_192_avx512 / aes_gcm_enc_256_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, ; u8 *out, @@ -3649,11 +3834,11 @@ FN_NAME(enc,_): INIT_GCM_MAX_LENGTH ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_enc ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_enc ;; Check IV != NULL @@ -3680,11 +3865,11 @@ FN_NAME(enc,_): ja error_enc ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 + or arg3, arg3 jz error_enc ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 + or arg4, arg4 jz error_enc skip_in_out_check_enc: @@ -3759,7 +3944,8 @@ skip_aad_check_error_enc: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_128_avx512 / aes_gcm_dec_192_avx512 / aes_gcm_dec_256_avx512 +;void aes_gcm_dec_128_avx_gen4 / aes_gcm_dec_192_avx_gen4 / aes_gcm_dec_256_avx_gen4 / +; aes_gcm_dec_128_avx512 / aes_gcm_dec_192_avx512 / aes_gcm_dec_256_avx512 ; (const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, ; u8 *out, @@ -3784,11 +3970,11 @@ FN_NAME(dec,_): INIT_GCM_MAX_LENGTH ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_dec ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_dec ;; Check IV != NULL @@ -3815,11 +4001,11 @@ FN_NAME(dec,_): ja error_dec ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 + or arg3, arg3 jz error_dec ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 + or arg4, arg4 jz error_dec skip_in_out_check_dec: @@ -3833,7 +4019,6 @@ skip_in_out_check_dec: skip_aad_check_dec: %endif - GCM_INIT arg1, arg2, arg6, arg7, arg8, r10, r11, r12 GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call @@ -3896,318 +4081,194 @@ skip_aad_check_error_dec: %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_enc_var_iv_128_avx512 / aes_gcm_enc_var_iv_192_avx512 / +; +; NOTE: THIS API IS USED BY JOB-API ONLY, NO NEED FOR 2ND SAFE PARAM CHECK +; +;IMB_JOB * aes_gcm_enc_var_iv_128_avx_gen4 / aes_gcm_enc_var_iv_192_avx_gen4 / +; aes_gcm_enc_var_iv_256_avx_gen4 / +; aes_gcm_enc_var_iv_128_avx512 / aes_gcm_enc_var_iv_192_avx512 / ; aes_gcm_enc_var_iv_256_avx512 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); +; (IMB_MGR *state, IMB_JOB *job) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(enc_var_iv,_),function,) +MKGLOBAL(FN_NAME(enc_var_iv,_),function,internal) FN_NAME(enc_var_iv,_): endbranch64 - FUNC_SAVE - -%ifdef SAFE_PARAM - ;; Reset imb_errno - IMB_ERR_CHECK_RESET - - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL - cmp arg1, 0 - jz error_enc_IV - - ;; Check context_data != NULL - cmp arg2, 0 - jz error_enc_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_enc_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_enc_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_enc_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_enc_IV - - cmp arg11, 16 - ja error_enc_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_enc_IV + FUNC_SAVE alloc_context - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_enc_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_enc_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_enc_IV + mov arg1, [arg2 + _enc_keys] -skip_in_out_check_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_enc_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_enc_IV - -skip_aad_check_enc_IV: -%endif - cmp arg7, 12 + cmp qword [arg2 + _iv_len_in_bytes], 12 je iv_len_12_enc_IV - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12, arg7 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]}, \ + r10, r11, r12, {[arg2 + _iv_len_in_bytes]} + jmp skip_iv_len_12_enc_IV iv_len_12_enc_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12 + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]}, \ + r10, r11, r12 skip_iv_len_12_enc_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, single_call + mov arg3, [arg2 + _src] + add arg3, [arg2 + _cipher_start_src_offset] + mov arg4, [arg2 + _dst] + mov [rsp + GP_OFFSET + 5*8], arg2 ; preserve job pointer + mov arg2, [arg2 + _msg_len_to_cipher] + GCM_ENC_DEC arg1, {rsp + CONTEXT_OFFSET}, arg4, arg3, arg2, ENC, single_call - GCM_COMPLETE arg1, arg2, arg10, arg11, single_call - -exit_enc_IV: - FUNC_RESTORE - - ret + mov arg2, [rsp + GP_OFFSET + 5*8] + GCM_COMPLETE arg1, {rsp + CONTEXT_OFFSET}, \ + {[arg2 + _auth_tag_output]}, {[arg2 + _auth_tag_output_len_in_bytes]}, \ + single_call -%ifdef SAFE_PARAM -error_enc_IV: - ;; Clear reg and imb_errno - IMB_ERR_CHECK_START rax + ;; mark job complete + mov dword [arg2 + _status], IMB_STATUS_COMPLETED - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY + mov rax, arg2 ;; return the job - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX + FUNC_RESTORE + ret - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; NOTE: THIS API IS USED BY JOB-API ONLY, NO NEED FOR 2ND SAFE PARAM CHECK +; +;IMB_JOB *aes_gcm_dec_var_iv_128_avx_gen4 / aes_gcm_dec_var_iv_192_avx_gen4 / +; aes_gcm_dec_var_iv_256_avx_gen4 / +; aes_gcm_dec_var_iv_128_avx512 / aes_gcm_dec_var_iv_192_avx512 / +; aes_gcm_dec_var_iv_256_avx512 +; (IMB_MGR *state, IMB_JOB *job) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(dec_var_iv,_),function,internal) +FN_NAME(dec_var_iv,_): + endbranch64 + FUNC_SAVE alloc_context - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN + mov arg1, [arg2 + _dec_keys] - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN + cmp qword [arg2 + _iv_len_in_bytes], 12 + je iv_len_12_dec_IV - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]}, \ + r10, r11, r12, {[arg2 + _iv_len_in_bytes]} - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_enc_IV + jmp skip_iv_len_12_dec_IV - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN +iv_len_12_dec_IV: + GCM_INIT arg1, {rsp + CONTEXT_OFFSET}, {[arg2 + _iv]}, \ + {[arg2 + _gcm_aad]}, {[arg2 + _gcm_aad_len]}, \ + r10, r11, r12 - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST +skip_iv_len_12_dec_IV: + mov arg3, [arg2 + _src] + add arg3, [arg2 + _cipher_start_src_offset] + mov arg4, [arg2 + _dst] + mov [rsp + GP_OFFSET + 5*8], arg2 ; preserve job pointer + mov arg2, [arg2 + _msg_len_to_cipher] + GCM_ENC_DEC arg1, {rsp + CONTEXT_OFFSET}, arg4, arg3, arg2, DEC, single_call - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC + mov arg2, [rsp + GP_OFFSET + 5*8] + GCM_COMPLETE arg1, {rsp + CONTEXT_OFFSET}, \ + {[arg2 + _auth_tag_output]}, {[arg2 + _auth_tag_output_len_in_bytes]}, \ + single_call -skip_in_out_check_error_enc_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_enc_IV + ;; mark job complete + mov dword [arg2 + _status], IMB_STATUS_COMPLETED - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD + mov rax, arg2 ;; return the job -skip_aad_check_error_enc_IV: - ;; Set imb_errno - IMB_ERR_CHECK_END rax - jmp exit_enc_IV -%endif + FUNC_RESTORE + ret -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void aes_gcm_dec_var_iv_128_avx512 / aes_gcm_dec_var_iv_192_avx512 / -; aes_gcm_dec_var_iv_256_avx512 -; (const struct gcm_key_data *key_data, -; struct gcm_context_data *context_data, -; u8 *out, -; const u8 *in, -; u64 msg_len, -; u8 *iv, -; const u64 iv_len, -; const u8 *aad, -; const u64 aad_len, -; u8 *auth_tag, -; const u64 auth_tag_len); -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(FN_NAME(dec_var_iv,_),function,) -FN_NAME(dec_var_iv,_): +%ifdef GCM128_MODE +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void ghash_pre_avx_gen4 / ghash_pre_avx512 +; (const void *key, struct gcm_key_data *key_data) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(GHASH_FN_NAME(ghash_pre),function,) +GHASH_FN_NAME(ghash_pre): endbranch64 - FUNC_SAVE - +;; Parameter is passed through register %ifdef SAFE_PARAM ;; Reset imb_errno IMB_ERR_CHECK_RESET - ;; Load max len to reg on windows - INIT_GCM_MAX_LENGTH - - ;; Check key_data != NULL + ;; Check key != NULL cmp arg1, 0 - jz error_dec_IV + jz error_ghash_pre - ;; Check context_data != NULL + ;; Check key_data != NULL cmp arg2, 0 - jz error_dec_IV - - ;; Check IV != NULL - cmp arg6, 0 - jz error_dec_IV - - ;; Check IV len != 0 - cmp arg7, 0 - jz error_dec_IV - - ;; Check auth_tag != NULL - cmp arg10, 0 - jz error_dec_IV - - ;; Check auth_tag_len == 0 or > 16 - cmp arg11, 0 - jz error_dec_IV - - cmp arg11, 16 - ja error_dec_IV - - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_dec_IV - - ;; Check if msg_len > max_len - cmp arg5, GCM_MAX_LENGTH - ja error_dec_IV - - ;; Check out != NULL (msg_len != 0) - cmp arg3, 0 - jz error_dec_IV - - ;; Check in != NULL (msg_len != 0) - cmp arg4, 0 - jz error_dec_IV - -skip_in_out_check_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_dec_IV - - ;; Check aad != NULL (aad_len != 0) - cmp arg8, 0 - jz error_dec_IV - -skip_aad_check_dec_IV: + jz error_ghash_pre %endif - cmp arg7, 12 - je iv_len_12_dec_IV - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12, arg7 - jmp skip_iv_len_12_dec_IV +%ifidn __OUTPUT_FORMAT__, win64 + sub rsp, 1*16 -iv_len_12_dec_IV: - GCM_INIT arg1, arg2, arg6, arg8, arg9, r10, r11, r12 + ; only xmm6 needs to be maintained + vmovdqu [rsp + 0*16], xmm6 +%endif + vmovdqu xmm6, [arg1] + vpshufb xmm6, [rel SHUF_MASK] + ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; + vmovdqa xmm2, xmm6 + vpsllq xmm6, xmm6, 1 + vpsrlq xmm2, xmm2, 63 + vmovdqa xmm1, xmm2 + vpslldq xmm2, xmm2, 8 + vpsrldq xmm1, xmm1, 8 + vpor xmm6, xmm6, xmm2 + ;reduction + vpshufd xmm2, xmm1, 00100100b + vpcmpeqd xmm2, [rel TWOONE] + vpand xmm2, xmm2, [rel POLY] + vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu [arg2 + HashKey], xmm6 ; store HashKey<<1 mod poly -skip_iv_len_12_dec_IV: - GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call - GCM_COMPLETE arg1, arg2, arg10, arg11, single_call + PRECOMPUTE arg2, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 -exit_dec_IV: - FUNC_RESTORE +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_xmms_avx_asm +%endif +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm6, [rsp + 0*16] + add rsp, 1*16 +%endif +exit_ghash_pre: ret %ifdef SAFE_PARAM -error_dec_IV: +error_ghash_pre: ;; Clear reg and imb_errno IMB_ERR_CHECK_START rax - ;; Check key_data != NULL - IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_EXP_KEY - - ;; Check context_data != NULL - IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_CTX - - ;; Check IV != NULL - IMB_ERR_CHECK_NULL arg6, rax, IMB_ERR_NULL_IV - - ;; Check IV len != 0 - IMB_ERR_CHECK_ZERO arg7, rax, IMB_ERR_IV_LEN - - ;; Check auth_tag != NULL - IMB_ERR_CHECK_NULL arg10, rax, IMB_ERR_NULL_AUTH - - ;; Check auth_tag_len == 0 or > 16 - IMB_ERR_CHECK_ZERO arg11, rax, IMB_ERR_AUTH_TAG_LEN - - IMB_ERR_CHECK_ABOVE arg11, 16, rax, IMB_ERR_AUTH_TAG_LEN + ;; Check key != NULL + IMB_ERR_CHECK_NULL arg1, rax, IMB_ERR_NULL_KEY - ;; Check if msg_len == 0 - cmp arg5, 0 - jz skip_in_out_check_error_dec_IV - - ;; Check if msg_len > max_len - IMB_ERR_CHECK_ABOVE arg5, GCM_MAX_LENGTH, rax, IMB_ERR_CIPH_LEN - - ;; Check out != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg3, rax, IMB_ERR_NULL_DST - - ;; Check in != NULL (msg_len != 0) - IMB_ERR_CHECK_NULL arg4, rax, IMB_ERR_NULL_SRC - -skip_in_out_check_error_dec_IV: - ;; Check if aad_len == 0 - cmp arg9, 0 - jz skip_aad_check_error_dec_IV - - ;; Check aad != NULL (aad_len != 0) - IMB_ERR_CHECK_NULL arg8, rax, IMB_ERR_NULL_AAD + ;; Check key_data != NULL + IMB_ERR_CHECK_NULL arg2, rax, IMB_ERR_NULL_EXP_KEY -skip_aad_check_error_dec_IV: ;; Set imb_errno IMB_ERR_CHECK_END rax - jmp exit_dec_IV + jmp exit_ghash_pre %endif -%ifdef GCM128_MODE ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void ghash_avx512 +;void ghash_avx_gen4 / ghash_avx512 ( ; const struct gcm_key_data *key_data, ; const void *in, ; const u64 in_len, -; void *tag, +; void *io_tag, ; const u64 tag_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -MKGLOBAL(ghash_avx512,function,) -ghash_avx512: +MKGLOBAL(GHASH_FN_NAME(ghash),function,) +GHASH_FN_NAME(ghash): endbranch64 FUNC_SAVE @@ -4216,19 +4277,19 @@ ghash_avx512: IMB_ERR_CHECK_RESET ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_ghash ;; Check in != NULL - cmp arg2, 0 + or arg2, arg2 jz error_ghash ;; Check in_len != 0 - cmp arg3, 0 + or arg3, arg3 jz error_ghash ;; Check tag != NULL - cmp arg4, 0 + or arg4, arg4 jz error_ghash ;; Check tag_len != 0 @@ -4240,9 +4301,13 @@ ghash_avx512: vmovdqu xmm0, [arg4] vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap +%ifdef IS_AVX2_GCM + CALC_AAD_HASH arg2, arg3, xmm0, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ + r10, r11, r12, r13, rax +%else CALC_AAD_HASH arg2, arg3, xmm0, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ r10, r11, r12 - +%endif vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap simd_store_avx arg4, xmm0, arg5, r12, rax @@ -4277,7 +4342,7 @@ error_ghash: jmp exit_ghash %endif -%endif +%endif ;; GCM128_MODE ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; PARTIAL_BLOCK_GMAC: Handles the tag partial blocks between update calls. @@ -4296,13 +4361,25 @@ error_ghash: %define %%HASH_SUBKEY %6 mov r13, [%%GDATA_CTX + PBlockLen] - cmp r13, 0 + or r13, r13 ; Leave Macro if no partial blocks je %%_partial_block_done ; Read in input data without over reading +%ifdef IS_AVX2_GCM + cmp %%PLAIN_LEN, 16 + jl %%_fewer_than_16_bytes + ; If more than 16 bytes of data, just fill the xmm register + VXLDR xmm1, [%%PLAIN_IN] + jmp %%_data_read + +%%_fewer_than_16_bytes: + lea r10, [%%PLAIN_IN] + READ_SMALL_DATA_INPUT_AVX xmm1, r10, %%PLAIN_LEN, rax +%else + ; Read in input data without over reading READ_SMALL_DATA_INPUT_LEN_BT16_AVX512 xmm1, %%PLAIN_IN, %%PLAIN_LEN, r12, rax, k1 - +%endif ; Finished reading in data %%_data_read: @@ -4349,7 +4426,7 @@ error_ghash: %%_ghash_done: vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH - cmp r15, 0 + or r15, r15 jl %%_partial_fill mov r12, 16 @@ -4365,8 +4442,10 @@ error_ghash: %endmacro ; PARTIAL_BLOCK_GMAC ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void imb_aes_gmac_update_128_avx512 / imb_aes_gmac_update_192_avx512 / -; imb_aes_gmac_update_256_avx512 +;void imb_aes_gmac_update_128_avx_gen4 / imb_aes_gmac_update_192_avx_gen4 / +; imb_aes_gmac_update_256_avx_gen4 +; imb_aes_gmac_update_128_avx512 / imb_aes_gmac_update_192_avx512 / +; imb_aes_gmac_update_256_avx512 ( ; const struct gcm_key_data *key_data, ; struct gcm_context_data *context_data, ; const u8 *in, @@ -4382,20 +4461,20 @@ GMAC_FN_NAME(update): IMB_ERR_CHECK_RESET %endif ;; Check if msg_len == 0 - cmp arg4, 0 - je exit_gmac_update + or arg4, arg4 + je exit_gmac_update %ifdef SAFE_PARAM ;; Check key_data != NULL - cmp arg1, 0 + or arg1, arg1 jz error_gmac_update ;; Check context_data != NULL - cmp arg2, 0 + or arg2, arg2 jz error_gmac_update ;; Check in != NULL (msg_len != 0) - cmp arg3, 0 + or arg3, arg3 jz error_gmac_update %endif @@ -4420,9 +4499,14 @@ GMAC_FN_NAME(update): jz no_full_blocks ;; Calculate GHASH of this segment +%ifdef IS_AVX2_GCM + CALC_AAD_HASH arg3, arg4, xmm8, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ + r10, r11, r12, r13, rax +%else CALC_AAD_HASH arg3, arg4, xmm8, arg1, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ r10, r11, r12 - vmovdqu [arg2 + AadHash], xmm8 ; ctx_data.aad hash = aad_hash +%endif + vmovdqu [arg2 + AadHash], xmm8 ; ctx_data.aad hash = aad_hash no_full_blocks: add arg3, arg4 ; Point at partial block @@ -4433,7 +4517,11 @@ no_full_blocks: ; Save next partial block mov [arg2 + PBlockLen], arg4 +%ifdef IS_AVX2_GCM + READ_SMALL_DATA_INPUT_AVX xmm1, arg3, arg4, r11 +%else READ_SMALL_DATA_INPUT_AVX512 xmm1, arg3, arg4, r11, k1 +%endif vpshufb xmm1, [rel SHUF_MASK] vpxor xmm8, xmm1 vmovdqu [arg2 + AadHash], xmm8 diff --git a/lib/include/gcm_defines.asm b/lib/include/gcm_defines.inc similarity index 91% rename from lib/include/gcm_defines.asm rename to lib/include/gcm_defines.inc index d1e4bb850234aa3e375ce2f05c3ed63bd73ba209..bc405ebcb142105069c8f9f3e9618c559ed27229 100644 --- a/lib/include/gcm_defines.asm +++ b/lib/include/gcm_defines.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,8 +25,8 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef GCM_DEFINES_ASM_INCLUDED -%define GCM_DEFINES_ASM_INCLUDED +%ifndef GCM_DEFINES_INC_INCLUDED +%define GCM_DEFINES_INC_INCLUDED ; ; Authors: @@ -37,8 +37,12 @@ mksection .rodata default rel -align 16 -POLY: dq 0x0000000000000001, 0xC200000000000000 +align 64 +POLY: + dq 0x0000000000000001, 0xC200000000000000 + dq 0x0000000000000001, 0xC200000000000000 + dq 0x0000000000000001, 0xC200000000000000 + dq 0x0000000000000001, 0xC200000000000000 align 64 POLY2: @@ -240,13 +244,13 @@ mksection .text %xdefine arg2 rdx %xdefine arg3 r8 %xdefine arg4 r9 - %xdefine arg5 qword [r14 + STACK_OFFSET + 8*5] - %xdefine arg6 qword [r14 + STACK_OFFSET + 8*6] - %xdefine arg7 qword [r14 + STACK_OFFSET + 8*7] - %xdefine arg8 qword [r14 + STACK_OFFSET + 8*8] - %xdefine arg9 qword [r14 + STACK_OFFSET + 8*9] - %xdefine arg10 qword [r14 + STACK_OFFSET + 8*10] - %xdefine arg11 qword [r14 + STACK_OFFSET + 8*11] + %xdefine arg5 qword [r14 + 8*5] + %xdefine arg6 qword [r14 + 8*6] + %xdefine arg7 qword [r14 + 8*7] + %xdefine arg8 qword [r14 + 8*8] + %xdefine arg9 qword [r14 + 8*9] + %xdefine arg10 qword [r14 + 8*10] + %xdefine arg11 qword [r14 + 8*11] %else %xdefine arg1 rdi %xdefine arg2 rsi @@ -254,11 +258,11 @@ mksection .text %xdefine arg4 rcx %xdefine arg5 r8 %xdefine arg6 r9 - %xdefine arg7 qword [r14 + STACK_OFFSET + 8*1] - %xdefine arg8 qword [r14 + STACK_OFFSET + 8*2] - %xdefine arg9 qword [r14 + STACK_OFFSET + 8*3] - %xdefine arg10 qword [r14 + STACK_OFFSET + 8*4] - %xdefine arg11 qword [r14 + STACK_OFFSET + 8*5] + %xdefine arg7 qword [r14 + 8*1] + %xdefine arg8 qword [r14 + 8*2] + %xdefine arg9 qword [r14 + 8*3] + %xdefine arg10 qword [r14 + 8*4] + %xdefine arg11 qword [r14 + 8*5] %endif %ifdef NT_LDST @@ -288,4 +292,4 @@ mksection .text %define VX512STR vmovdqu8 %endif -%endif ; GCM_DEFINES_ASM_INCLUDED +%endif ; GCM_DEFINES_INC_INCLUDED diff --git a/lib/sse_t1/gcm_gmac_api_sse.inc b/lib/include/gcm_gmac_api_sse.inc similarity index 94% rename from lib/sse_t1/gcm_gmac_api_sse.inc rename to lib/include/gcm_gmac_api_sse.inc index da90159d2724c63b8848ff01fc5b379e13333ac1..f22fe3c6aa8cddc11dde345aa89ee833fbe2f95c 100644 --- a/lib/sse_t1/gcm_gmac_api_sse.inc +++ b/lib/include/gcm_gmac_api_sse.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "sse_t1/gcm_sse.inc" +%include "include/gcm_sse.inc" %include "include/gcm_common.inc" %ifndef GCM_GMAC_API_SSE_INC @@ -41,7 +41,10 @@ default rel ;void ghash_pre_sse / ghash_sse_no_aesni ; (const void *key, struct gcm_key_data *key_data) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 %ifndef NO_AESNI +MKGLOBAL(ghash_pre_avx_gen2,function,) +ghash_pre_avx_gen2: MKGLOBAL(ghash_pre_sse,function,) ghash_pre_sse: %else @@ -64,14 +67,9 @@ ghash_pre_sse_no_aesni: %endif %ifidn __OUTPUT_FORMAT__, win64 - push r14 - mov r14, rsp - - sub rsp, VARIABLE_OFFSET - and rsp, ~63 ; align rsp to 64 bytes - + sub rsp, 1*16 ; only xmm6 needs to be maintained - movdqa [rsp + LOCAL_STORAGE + 0*16],xmm6 + movdqu [rsp + 0*16], xmm6 %endif movdqu xmm6, [arg1] pshufb xmm6, [rel SHUF_MASK] @@ -98,10 +96,8 @@ ghash_pre_sse_no_aesni: clear_scratch_xmms_sse_asm %endif %ifidn __OUTPUT_FORMAT__, win64 - movdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] - mov rsp, r14 - pop r14 - + movdqu xmm6, [rsp + 0*16] + add rsp, 1*16 %endif exit_ghash_pre: ret @@ -130,7 +126,10 @@ error_ghash_pre: ; void *io_tag, ; const u64 tag_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 %ifndef NO_AESNI +MKGLOBAL(ghash_avx_gen2,function,) +ghash_avx_gen2: MKGLOBAL(ghash_sse,function,) ghash_sse: %else @@ -216,6 +215,11 @@ error_ghash: ; const u8 *in, ; const u64 msg_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(GMAC_FN_NAME_AVX(update),function,) +GMAC_FN_NAME_AVX(update): +%endif MKGLOBAL(GMAC_FN_NAME(update),function,) GMAC_FN_NAME(update): endbranch64 @@ -277,7 +281,7 @@ no_full_blocks: ; Save next partial block mov [arg2 + PBlockLen], arg4 - READ_SMALL_DATA_INPUT_SSE xmm1, arg3, arg4, r11, r12, r13 + READ_SMALL_DATA_INPUT_SSE xmm1, arg3, arg4, r11 pshufb xmm1, [rel SHUF_MASK] pxor xmm8, xmm1 movdqu [arg2 + AadHash], xmm8 diff --git a/lib/avx512_t2/gcm_gmac_api_vaes_avx512.inc b/lib/include/gcm_gmac_api_vaes_avx512.inc similarity index 86% rename from lib/avx512_t2/gcm_gmac_api_vaes_avx512.inc rename to lib/include/gcm_gmac_api_vaes_avx512.inc index 8ca8df8d5f30387165ddf8d60ca75f824055b995..b82e86da8d070015a9045d934a1f68edad387510 100644 --- a/lib/avx512_t2/gcm_gmac_api_vaes_avx512.inc +++ b/lib/include/gcm_gmac_api_vaes_avx512.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,8 +27,9 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512_t2/gcm_vaes_avx512.inc" +%include "include/gcm_vaes_avx512.inc" %include "include/error.inc" +%include "include/clear_regs.inc" %ifndef GCM_GMAC_API_VAES_AVX512_INC %define GCM_GMAC_API_VAES_AVX512_INC @@ -38,7 +39,7 @@ default rel %ifdef GCM128_MODE ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;void ghash_pre_avx512 +;void ghash_pre_vaes_avx512 ; (const void *key, struct gcm_key_data *key_data) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(ghash_pre_vaes_avx512,function,) @@ -58,7 +59,7 @@ ghash_pre_vaes_avx512: jz error_ghash_pre %endif - FUNC_SAVE + FUNC_SAVE small_frame vmovdqu xmm6, [arg1] vpshufb xmm6, [rel SHUF_MASK] @@ -76,10 +77,12 @@ ghash_pre_vaes_avx512: vpand xmm2, xmm2, [rel POLY] vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - vmovdqu [arg2 + HashKey], xmm6 ; store HashKey<<1 mod poly + vmovdqu [arg2 + HashKey_1], xmm6 ; store HashKey<<1 mod poly PRECOMPUTE arg2, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm7, xmm8 - +%ifdef SAFE_DATA + clear_zmms_avx512 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 +%endif FUNC_RESTORE exit_ghash_pre: @@ -112,7 +115,7 @@ error_ghash_pre: MKGLOBAL(ghash_vaes_avx512,function,) ghash_vaes_avx512: endbranch64 - FUNC_SAVE + FUNC_SAVE small_frame %ifdef SAFE_PARAM ;; Reset imb_errno @@ -141,16 +144,19 @@ ghash_vaes_avx512: ;; copy tag to xmm0 vmovdqu xmm0, [arg4] - vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb xmm0, xmm0, [rel SHUF_MASK] ; perform a 16Byte swap CALC_AAD_HASH arg2, arg3, xmm0, arg1, zmm1, zmm2, zmm3, zmm4, zmm5, \ zmm6, zmm7, zmm8, zmm9, zmm10, zmm11, zmm12, zmm13, \ zmm15, zmm16, zmm17, zmm18, zmm19, r10, r11, r12, k1 + ;; **zmm2, zmm3, zmm4 and zmm5 may contain clear text + ;; **zmm13, zmm15, zmm18 and zmm8 may contain authentication key - vpshufb xmm0, [rel SHUF_MASK] ; perform a 16Byte swap - + vpshufb xmm0, xmm0, [rel SHUF_MASK] ; perform a 16Byte swap simd_store_avx arg4, xmm0, arg5, r12, rax - +%ifdef SAFE_DATA + clear_zmms_avx512 xmm0, xmm2, xmm3, xmm4, xmm5, xmm13, xmm15, xmm8, xmm18 +%endif exit_ghash: FUNC_RESTORE ret @@ -195,7 +201,7 @@ error_ghash: MKGLOBAL(GMAC_FN_NAME(update),function,) GMAC_FN_NAME(update): endbranch64 - FUNC_SAVE + FUNC_SAVE small_frame %ifdef SAFE_PARAM ;; Reset imb_errno @@ -227,9 +233,10 @@ GMAC_FN_NAME(update): vmovdqu64 xmm8, [arg2 + AadHash] PARTIAL_BLOCK_GMAC arg1, arg2, arg3, arg4, r11, xmm8, r10, r12, rax, \ - zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, \ - zmm9, zmm10, k1 - + zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, k1 +%ifdef SAFE_DATA + clear_zmms_avx512 xmm0 +%endif ; CALC_AAD_HASH needs to deal with multiple of 16 bytes sub arg4, r11 add arg3, r11 @@ -246,6 +253,12 @@ GMAC_FN_NAME(update): zmm16, zmm17, zmm18, zmm19, zmm20, r10, r11, r12, k1 vmovdqu64 [arg2 + AadHash], xmm8 ; ctx_data.aad hash = aad_hash + ;; **zmm2, zmm3, zmm4 and zmm5 may contain clear text + ;; **zmm19 may contain authentication key +%ifdef SAFE_DATA + clear_zmms_avx512 xmm2, xmm3, xmm4, xmm5, xmm18, xmm19 +%endif + no_full_blocks: add arg3, arg4 ; Point at partial block @@ -256,13 +269,15 @@ no_full_blocks: ; Save next partial block mov [arg2 + PBlockLen], arg4 READ_SMALL_DATA_INPUT_AVX512 xmm1, arg3, arg4, r11, k1 - vpshufb xmm1, [rel SHUF_MASK] - vpxorq xmm8, xmm1 + vpshufb xmm1, xmm1, [rel SHUF_MASK] + vpxorq xmm8, xmm8, xmm1 vmovdqu64 [arg2 + AadHash], xmm8 - +%ifdef SAFE_DATA + ;; **xmm1 and xmm8 may contain some clear text + clear_zmms_avx512 xmm1, xmm8 +%endif exit_gmac_update: - FUNC_RESTORE - + FUNC_RESTORE ret %ifdef SAFE_PARAM diff --git a/lib/include/gcm_keys_avx2_avx512.asm b/lib/include/gcm_keys_avx2_avx512.inc similarity index 98% rename from lib/include/gcm_keys_avx2_avx512.asm rename to lib/include/gcm_keys_avx2_avx512.inc index 64fec024048bd04e9e696feaa0feb1a94d457d70..e9605c6751900631293d63c9a6a41e034f664be6 100644 --- a/lib/include/gcm_keys_avx2_avx512.asm +++ b/lib/include/gcm_keys_avx2_avx512.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/gcm_keys_sse_avx.asm b/lib/include/gcm_keys_sse_avx.inc similarity index 99% rename from lib/include/gcm_keys_sse_avx.asm rename to lib/include/gcm_keys_sse_avx.inc index 16eb60d10d5113ad32b1f1eff36c4309a06aa1fa..5e9eea7a3b47cfeee00aa3dfa85c4278a8a3deef 100644 --- a/lib/include/gcm_keys_sse_avx.asm +++ b/lib/include/gcm_keys_sse_avx.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/include/gcm_keys_vaes_avx512.asm b/lib/include/gcm_keys_vaes_avx512.asm deleted file mode 100644 index 4db9b089c59fe7b3e9de1e18333174552dc99145..0000000000000000000000000000000000000000 --- a/lib/include/gcm_keys_vaes_avx512.asm +++ /dev/null @@ -1,96 +0,0 @@ -;; -;; Copyright (c) 2019-2022, Intel Corporation -;; -;; Redistribution and use in source and binary forms, with or without -;; modification, are permitted provided that the following conditions are met: -;; -;; * Redistributions of source code must retain the above copyright notice, -;; this list of conditions and the following disclaimer. -;; * Redistributions in binary form must reproduce the above copyright -;; notice, this list of conditions and the following disclaimer in the -;; documentation and/or other materials provided with the distribution. -;; * Neither the name of Intel Corporation nor the names of its contributors -;; may be used to endorse or promote products derived from this software -;; without specific prior written permission. -;; -;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;; - -%ifndef GCM_KEYS_VAES_AVX512_INCLUDED -%define GCM_KEYS_VAES_AVX512_INCLUDED - -;; Define the fields of gcm_key_data struct: -;; uint8_t expanded_keys[GCM_ENC_KEY_LEN * GCM_KEY_SETS]; -;; uint8_t shifted_hkey_9_128[GCM_ENC_KEY_LEN * (128 - 8)]; -;; uint8_t shifted_hkey_8[GCM_ENC_KEY_LEN]; // HashKey^8 <<1 mod poly -;; uint8_t shifted_hkey_7[GCM_ENC_KEY_LEN]; // HashKey^7 <<1 mod poly -;; uint8_t shifted_hkey_6[GCM_ENC_KEY_LEN]; // HashKey^6 <<1 mod poly -;; uint8_t shifted_hkey_5[GCM_ENC_KEY_LEN]; // HashKey^5 <<1 mod poly -;; uint8_t shifted_hkey_4[GCM_ENC_KEY_LEN]; // HashKey^4 <<1 mod poly -;; uint8_t shifted_hkey_3[GCM_ENC_KEY_LEN]; // HashKey^3 <<1 mod poly -;; uint8_t shifted_hkey_2[GCM_ENC_KEY_LEN]; // HashKey^2 <<1 mod poly -;; uint8_t shifted_hkey_1[GCM_ENC_KEY_LEN]; // HashKey <<1 mod poly - -;; -;; Key structure holds up to 48 ghash keys -;; -%define HashKey_48 (16*15) ; HashKey^48 <<1 mod poly -%define HashKey_47 (16*16) ; HashKey^47 <<1 mod poly -%define HashKey_46 (16*17) ; HashKey^46 <<1 mod poly -%define HashKey_45 (16*18) ; HashKey^45 <<1 mod poly -%define HashKey_44 (16*19) ; HashKey^44 <<1 mod poly -%define HashKey_43 (16*20) ; HashKey^43 <<1 mod poly -%define HashKey_42 (16*21) ; HashKey^42 <<1 mod poly -%define HashKey_41 (16*22) ; HashKey^41 <<1 mod poly -%define HashKey_40 (16*23) ; HashKey^40 <<1 mod poly -%define HashKey_39 (16*24) ; HashKey^39 <<1 mod poly -%define HashKey_38 (16*25) ; HashKey^38 <<1 mod poly -%define HashKey_37 (16*26) ; HashKey^37 <<1 mod poly -%define HashKey_36 (16*27) ; HashKey^36 <<1 mod poly -%define HashKey_35 (16*28) ; HashKey^35 <<1 mod poly -%define HashKey_34 (16*29) ; HashKey^34 <<1 mod poly -%define HashKey_33 (16*30) ; HashKey^33 <<1 mod poly -%define HashKey_32 (16*31) ; HashKey^32 <<1 mod poly -%define HashKey_31 (16*32) ; HashKey^31 <<1 mod poly -%define HashKey_30 (16*33) ; HashKey^30 <<1 mod poly -%define HashKey_29 (16*34) ; HashKey^29 <<1 mod poly -%define HashKey_28 (16*35) ; HashKey^28 <<1 mod poly -%define HashKey_27 (16*36) ; HashKey^27 <<1 mod poly -%define HashKey_26 (16*37) ; HashKey^26 <<1 mod poly -%define HashKey_25 (16*38) ; HashKey^25 <<1 mod poly -%define HashKey_24 (16*39) ; HashKey^24 <<1 mod poly -%define HashKey_23 (16*40) ; HashKey^23 <<1 mod poly -%define HashKey_22 (16*41) ; HashKey^22 <<1 mod poly -%define HashKey_21 (16*42) ; HashKey^21 <<1 mod poly -%define HashKey_20 (16*43) ; HashKey^20 <<1 mod poly -%define HashKey_19 (16*44) ; HashKey^19 <<1 mod poly -%define HashKey_18 (16*45) ; HashKey^18 <<1 mod poly -%define HashKey_17 (16*46) ; HashKey^17 <<1 mod poly -%define HashKey_16 (16*47) ; HashKey^16 <<1 mod poly -%define HashKey_15 (16*48) ; HashKey^15 <<1 mod poly -%define HashKey_14 (16*49) ; HashKey^14 <<1 mod poly -%define HashKey_13 (16*50) ; HashKey^13 <<1 mod poly -%define HashKey_12 (16*51) ; HashKey^12 <<1 mod poly -%define HashKey_11 (16*52) ; HashKey^11 <<1 mod poly -%define HashKey_10 (16*53) ; HashKey^10 <<1 mod poly -%define HashKey_9 (16*54) ; HashKey^9 <<1 mod poly -%define HashKey_8 (16*55) ; HashKey^8 <<1 mod poly -%define HashKey_7 (16*56) ; HashKey^7 <<1 mod poly -%define HashKey_6 (16*57) ; HashKey^6 <<1 mod poly -%define HashKey_5 (16*58) ; HashKey^5 <<1 mod poly -%define HashKey_4 (16*59) ; HashKey^4 <<1 mod poly -%define HashKey_3 (16*60) ; HashKey^3 <<1 mod poly -%define HashKey_2 (16*61) ; HashKey^2 <<1 mod poly -%define HashKey_1 (16*62) ; HashKey <<1 mod poly -%define HashKey (16*62) ; HashKey <<1 mod poly - -%endif ; GCM_KEYS_VAES_AVX512_INCLUDED diff --git a/lib/include/gcm_keys_vaes_avx512.inc b/lib/include/gcm_keys_vaes_avx512.inc new file mode 100644 index 0000000000000000000000000000000000000000..5419f4d69b0af2eabca1411d01be351b1b7d5a59 --- /dev/null +++ b/lib/include/gcm_keys_vaes_avx512.inc @@ -0,0 +1,106 @@ +;; +;; Copyright (c) 2019-2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%ifndef GCM_KEYS_VAES_AVX512_INCLUDED +%define GCM_KEYS_VAES_AVX512_INCLUDED + +;; First 15 128-bit words are reserved for AES round keys +%xdefine HKeyStart (16 * 15) + +;; +;; Key structure holds up to 32 ghash keys +;; +%xdefine HashKey_32 ((16 * 0) + HKeyStart) ; HashKey^32 <<1 mod poly +%xdefine HashKey_31 ((16 * 1) + HKeyStart) ; HashKey^31 <<1 mod poly +%xdefine HashKey_30 ((16 * 2) + HKeyStart) ; HashKey^30 <<1 mod poly +%xdefine HashKey_29 ((16 * 3) + HKeyStart) ; HashKey^29 <<1 mod poly +%xdefine HashKey_28 ((16 * 4) + HKeyStart) ; HashKey^28 <<1 mod poly +%xdefine HashKey_27 ((16 * 5) + HKeyStart) ; HashKey^27 <<1 mod poly +%xdefine HashKey_26 ((16 * 6) + HKeyStart) ; HashKey^26 <<1 mod poly +%xdefine HashKey_25 ((16 * 7) + HKeyStart) ; HashKey^25 <<1 mod poly +%xdefine HashKey_24 ((16 * 8) + HKeyStart) ; HashKey^24 <<1 mod poly +%xdefine HashKey_23 ((16 * 9) + HKeyStart) ; HashKey^23 <<1 mod poly +%xdefine HashKey_22 ((16 * 10) + HKeyStart) ; HashKey^22 <<1 mod poly +%xdefine HashKey_21 ((16 * 11) + HKeyStart) ; HashKey^21 <<1 mod poly +%xdefine HashKey_20 ((16 * 12) + HKeyStart) ; HashKey^20 <<1 mod poly +%xdefine HashKey_19 ((16 * 13) + HKeyStart) ; HashKey^19 <<1 mod poly +%xdefine HashKey_18 ((16 * 14) + HKeyStart) ; HashKey^18 <<1 mod poly +%xdefine HashKey_17 ((16 * 15) + HKeyStart) ; HashKey^17 <<1 mod poly +%xdefine HashKey_16 ((16 * 16) + HKeyStart) ; HashKey^16 <<1 mod poly +%xdefine HashKey_15 ((16 * 17) + HKeyStart) ; HashKey^15 <<1 mod poly +%xdefine HashKey_14 ((16 * 18) + HKeyStart) ; HashKey^14 <<1 mod poly +%xdefine HashKey_13 ((16 * 19) + HKeyStart) ; HashKey^13 <<1 mod poly +%xdefine HashKey_12 ((16 * 20) + HKeyStart) ; HashKey^12 <<1 mod poly +%xdefine HashKey_11 ((16 * 21) + HKeyStart) ; HashKey^11 <<1 mod poly +%xdefine HashKey_10 ((16 * 22) + HKeyStart) ; HashKey^10 <<1 mod poly +%xdefine HashKey_9 ((16 * 23) + HKeyStart) ; HashKey^9 <<1 mod poly +%xdefine HashKey_8 ((16 * 24) + HKeyStart) ; HashKey^8 <<1 mod poly +%xdefine HashKey_7 ((16 * 25) + HKeyStart) ; HashKey^7 <<1 mod poly +%xdefine HashKey_6 ((16 * 26) + HKeyStart) ; HashKey^6 <<1 mod poly +%xdefine HashKey_5 ((16 * 27) + HKeyStart) ; HashKey^5 <<1 mod poly +%xdefine HashKey_4 ((16 * 28) + HKeyStart) ; HashKey^4 <<1 mod poly +%xdefine HashKey_3 ((16 * 29) + HKeyStart) ; HashKey^3 <<1 mod poly +%xdefine HashKey_2 ((16 * 30) + HKeyStart) ; HashKey^2 <<1 mod poly +%xdefine HashKey_1 ((16 * 31) + HKeyStart) ; HashKey <<1 mod poly + +%xdefine HKeyGap (16 * 32) +;; (HashKey^n mode POLY) x POLY constants + +%xdefine HashKeyK_32 (HashKey_32 + HKeyGap) ; (HashKey^32 <<1 mod poly) x POLY +%xdefine HashKeyK_31 (HashKey_31 + HKeyGap) ; (HashKey^31 <<1 mod poly) x POLY +%xdefine HashKeyK_30 (HashKey_30 + HKeyGap) ; (HashKey^30 <<1 mod poly) x POLY +%xdefine HashKeyK_29 (HashKey_29 + HKeyGap) ; (HashKey^29 <<1 mod poly) x POLY +%xdefine HashKeyK_28 (HashKey_28 + HKeyGap) ; (HashKey^28 <<1 mod poly) x POLY +%xdefine HashKeyK_27 (HashKey_27 + HKeyGap) ; (HashKey^27 <<1 mod poly) x POLY +%xdefine HashKeyK_26 (HashKey_26 + HKeyGap) ; (HashKey^26 <<1 mod poly) x POLY +%xdefine HashKeyK_25 (HashKey_25 + HKeyGap) ; (HashKey^25 <<1 mod poly) x POLY +%xdefine HashKeyK_24 (HashKey_24 + HKeyGap) ; (HashKey^24 <<1 mod poly) x POLY +%xdefine HashKeyK_23 (HashKey_23 + HKeyGap) ; (HashKey^23 <<1 mod poly) x POLY +%xdefine HashKeyK_22 (HashKey_22 + HKeyGap) ; (HashKey^22 <<1 mod poly) x POLY +%xdefine HashKeyK_21 (HashKey_21 + HKeyGap) ; (HashKey^21 <<1 mod poly) x POLY +%xdefine HashKeyK_20 (HashKey_20 + HKeyGap) ; (HashKey^20 <<1 mod poly) x POLY +%xdefine HashKeyK_19 (HashKey_19 + HKeyGap) ; (HashKey^19 <<1 mod poly) x POLY +%xdefine HashKeyK_18 (HashKey_18 + HKeyGap) ; (HashKey^18 <<1 mod poly) x POLY +%xdefine HashKeyK_17 (HashKey_17 + HKeyGap) ; (HashKey^17 <<1 mod poly) x POLY +%xdefine HashKeyK_16 (HashKey_16 + HKeyGap) ; (HashKey^16 <<1 mod poly) x POLY +%xdefine HashKeyK_15 (HashKey_15 + HKeyGap) ; (HashKey^15 <<1 mod poly) x POLY +%xdefine HashKeyK_14 (HashKey_14 + HKeyGap) ; (HashKey^14 <<1 mod poly) x POLY +%xdefine HashKeyK_13 (HashKey_13 + HKeyGap) ; (HashKey^13 <<1 mod poly) x POLY +%xdefine HashKeyK_12 (HashKey_12 + HKeyGap) ; (HashKey^12 <<1 mod poly) x POLY +%xdefine HashKeyK_11 (HashKey_11 + HKeyGap) ; (HashKey^11 <<1 mod poly) x POLY +%xdefine HashKeyK_10 (HashKey_10 + HKeyGap) ; (HashKey^10 <<1 mod poly) x POLY +%xdefine HashKeyK_9 (HashKey_9 + HKeyGap) ; (HashKey^9 <<1 mod poly) x POLY +%xdefine HashKeyK_8 (HashKey_8 + HKeyGap) ; (HashKey^8 <<1 mod poly) x POLY +%xdefine HashKeyK_7 (HashKey_7 + HKeyGap) ; (HashKey^7 <<1 mod poly) x POLY +%xdefine HashKeyK_6 (HashKey_6 + HKeyGap) ; (HashKey^6 <<1 mod poly) x POLY +%xdefine HashKeyK_5 (HashKey_5 + HKeyGap) ; (HashKey^5 <<1 mod poly) x POLY +%xdefine HashKeyK_4 (HashKey_4 + HKeyGap) ; (HashKey^4 <<1 mod poly) x POLY +%xdefine HashKeyK_3 (HashKey_3 + HKeyGap) ; (HashKey^3 <<1 mod poly) x POLY +%xdefine HashKeyK_2 (HashKey_2 + HKeyGap) ; (HashKey^2 <<1 mod poly) x POLY +%xdefine HashKeyK_1 (HashKey_1 + HKeyGap) ; (HashKey <<1 mod poly) x POLY + +%endif ; GCM_KEYS_VAES_AVX512_INCLUDED diff --git a/lib/sse_t1/gcm_sgl_api_sse.inc b/lib/include/gcm_sgl_api_sse.inc similarity index 95% rename from lib/sse_t1/gcm_sgl_api_sse.inc rename to lib/include/gcm_sgl_api_sse.inc index 82f7b2b794e8f6dbf0cfc9f7bcd199b9f5746a78..a8b9fdabd3b724cc8c4c400d3c6d2502c07517d3 100644 --- a/lib/sse_t1/gcm_sgl_api_sse.inc +++ b/lib/include/gcm_sgl_api_sse.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "sse_t1/gcm_sse.inc" +%include "include/gcm_sse.inc" %ifndef GCM_SGL_API_SSE_INC %define GCM_SGL_API_SSE_INC @@ -43,6 +43,11 @@ default rel ; const u8 *aad, ; u64 aad_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(init,_),function,) +FN_NAME_AVX(init,_): +%endif MKGLOBAL(FN_NAME(init,_),function,) FN_NAME(init,_): endbranch64 @@ -51,7 +56,7 @@ FN_NAME(init,_): %ifidn __OUTPUT_FORMAT__, win64 push r14 push r15 - mov r14, rsp + lea r14, [rsp + 4*8] ; xmm6 needs to be maintained for Windows sub rsp, 1*16 movdqu [rsp + 0*16], xmm6 @@ -93,7 +98,7 @@ exit_init: %ifidn __OUTPUT_FORMAT__, win64 movdqu xmm6 , [rsp + 0*16] - mov rsp, r14 + add rsp, 16 pop r15 pop r14 %endif @@ -139,6 +144,11 @@ skip_aad_check_error_init: ; const u8 *aad, ; const u64 aad_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(init_var_iv,_),function,) +FN_NAME_AVX(init_var_iv,_): +%endif MKGLOBAL(FN_NAME(init_var_iv,_),function,) FN_NAME(init_var_iv,_): endbranch64 @@ -147,7 +157,7 @@ FN_NAME(init_var_iv,_): %ifidn __OUTPUT_FORMAT__, win64 push r14 push r15 - mov r14, rsp + lea r14, [rsp + 4*8] ; xmm6 needs to be maintained for Windows sub rsp, 1*16 movdqu [rsp + 0*16], xmm6 @@ -201,7 +211,7 @@ exit_init_IV: %ifidn __OUTPUT_FORMAT__, win64 movdqu xmm6 , [rsp + 0*16] - mov rsp, r14 + add rsp, 1*16 pop r15 pop r14 %endif @@ -249,6 +259,11 @@ skip_aad_check_error_init_IV: ; const u8 *in, ; u64 msg_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(enc,_update_),function,) +FN_NAME_AVX(enc,_update_): +%endif MKGLOBAL(FN_NAME(enc,_update_),function,) FN_NAME(enc,_update_): endbranch64 @@ -330,6 +345,11 @@ skip_in_out_check_error_update_enc: ; const u8 *in, ; u64 msg_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(dec,_update_),function,) +FN_NAME_AVX(dec,_update_): +%endif MKGLOBAL(FN_NAME(dec,_update_),function,) FN_NAME(dec,_update_): endbranch64 @@ -411,6 +431,11 @@ skip_in_out_check_error_update_dec: ; u8 *auth_tag, ; u64 auth_tag_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(enc,_finalize_),function,) +FN_NAME_AVX(enc,_finalize_): +%endif MKGLOBAL(FN_NAME(enc,_finalize_),function,) FN_NAME(enc,_finalize_): endbranch64 @@ -504,6 +529,11 @@ error_enc_fin: ; u8 *auth_tag, ; u64 auth_tag_len); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align 32 +%ifndef NO_AESNI +MKGLOBAL(FN_NAME_AVX(dec,_finalize_),function,) +FN_NAME_AVX(dec,_finalize_): +%endif MKGLOBAL(FN_NAME(dec,_finalize_),function,) FN_NAME(dec,_finalize_): endbranch64 diff --git a/lib/avx512_t2/gcm_sgl_api_vaes_avx512.inc b/lib/include/gcm_sgl_api_vaes_avx512.inc similarity index 95% rename from lib/avx512_t2/gcm_sgl_api_vaes_avx512.inc rename to lib/include/gcm_sgl_api_vaes_avx512.inc index 1ecc7ba19f0adcab7d50be1ecfd148da9d145880..ecdcf4cc9bd71a32b84c545bb6d5d38c7f2636d5 100644 --- a/lib/avx512_t2/gcm_sgl_api_vaes_avx512.inc +++ b/lib/include/gcm_sgl_api_vaes_avx512.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,9 +27,10 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512_t2/gcm_vaes_avx512.inc" +%include "include/gcm_vaes_avx512.inc" %include "include/error.inc" -%include "include/os.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef GCM_SGL_API_VAES_AVX512_INC %define GCM_SGL_API_VAES_AVX512_INC @@ -48,7 +49,7 @@ default rel MKGLOBAL(FN_NAME(init,_),function,) FN_NAME(init,_): endbranch64 - FUNC_SAVE + FUNC_SAVE small_frame %ifdef SAFE_PARAM ;; Reset imb_errno @@ -126,7 +127,7 @@ skip_aad_check_error_init: MKGLOBAL(FN_NAME(init_var_iv,_),function,) FN_NAME(init_var_iv,_): endbranch64 - FUNC_SAVE + FUNC_SAVE small_frame %ifdef SAFE_PARAM ;; Reset imb_errno @@ -256,6 +257,9 @@ FN_NAME(enc,_update_): jz error_update_enc %endif GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, multi_call +%ifdef SAFE_DATA + clear_zmms_avx512 xmm6 +%endif exit_update_enc: FUNC_RESTORE @@ -339,7 +343,9 @@ FN_NAME(dec,_update_): %endif GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, multi_call - +%ifdef SAFE_DATA + clear_zmms_avx512 xmm6 +%endif exit_update_dec: FUNC_RESTORE ret @@ -411,11 +417,13 @@ FN_NAME(enc,_finalize_): ja error_enc_fin %endif - FUNC_SAVE + FUNC_SAVE small_frame GCM_COMPLETE arg1, arg2, arg3, arg4, multi_call, k1, r10, r11, r12 - +%ifdef SAFE_DATA + ;; **xmm5, xmm6, xmm11, xmm13, xmm14 and xmm16 may contain sensitive data + clear_zmms_avx512 xmm5, xmm6, xmm11, xmm13, xmm14, xmm16 +%endif FUNC_RESTORE - exit_enc_fin: ret @@ -480,11 +488,15 @@ FN_NAME(dec,_finalize_): ja error_dec_fin %endif - FUNC_SAVE + FUNC_SAVE small_frame GCM_COMPLETE arg1, arg2, arg3, arg4, multi_call, k1, r10, r11, r12 - FUNC_RESTORE +%ifdef SAFE_DATA + ;; **xmm5, xmm6, xmm11, xmm13, xmm14 and xmm16 may contain sensitive data + clear_zmms_avx512 xmm5, xmm6, xmm11, xmm13, xmm14, xmm16 +%endif + FUNC_RESTORE exit_dec_fin: ret diff --git a/lib/sse_t1/gcm_sse.inc b/lib/include/gcm_sse.inc similarity index 90% rename from lib/sse_t1/gcm_sse.inc rename to lib/include/gcm_sse.inc index 042ef0737143c0e73c8df829fef12f1c6adfbe0a..f1b58ac48fc6069fa14360cdd0c79e1ef6190f03 100644 --- a/lib/sse_t1/gcm_sse.inc +++ b/lib/include/gcm_sse.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2011-2022 Intel Corporation All rights reserved. +; Copyright(c) 2011-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -114,13 +114,13 @@ %ifndef GCM_SSE_INC %define GCM_SSE_INC -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" -%include "include/gcm_defines.asm" -%include "include/gcm_keys_sse_avx.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" +%include "include/gcm_defines.inc" +%include "include/gcm_keys_sse_avx.inc" %include "include/gcm_common.inc" -%include "include/memcpy.asm" +%include "include/memcpy.inc" %include "include/cet.inc" %include "include/error.inc" %ifndef GCM128_MODE @@ -140,23 +140,35 @@ %ifdef GCM128_MODE %define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ SSE %define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _128_ %+ SSE +%ifndef NO_AESNI +%define FN_NAME_AVX(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ avx_gen2 +%define GMAC_FN_NAME_AVX(x) imb_aes_gmac_ %+ x %+ _128_avx_gen2 +%endif %define NROUNDS 9 %endif %ifdef GCM192_MODE %define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ SSE %define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _192_ %+ SSE +%ifndef NO_AESNI +%define FN_NAME_AVX(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ avx_gen2 +%define GMAC_FN_NAME_AVX(x) imb_aes_gmac_ %+ x %+ _192_avx_gen2 +%endif %define NROUNDS 11 %endif %ifdef GCM256_MODE %define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ SSE %define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _256_ %+ SSE +%ifndef NO_AESNI +%define FN_NAME_AVX(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ avx_gen2 +%define GMAC_FN_NAME_AVX(x) imb_aes_gmac_ %+ x %+ _256_avx_gen2 +%endif %define NROUNDS 13 %endif -; need to push 4 registers into stack to maintain -%define STACK_OFFSET 8*4 +; need to store 5 GP registers on stack (align to 16 bytes) +%define GP_STORAGE 8*6 %define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register) %define TMP3 16*1 ; Temporary storage for AES State 3 @@ -174,7 +186,25 @@ %define XMM_STORAGE 0 %endif -%define VARIABLE_OFFSET LOCAL_STORAGE + XMM_STORAGE +%define GP_OFFSET (LOCAL_STORAGE + XMM_STORAGE) + +%define VARIABLE_OFFSET (GP_STORAGE + LOCAL_STORAGE + XMM_STORAGE) + +;; extra memory for GCM context structure +%define CONTEXT_SIZE 6*16 +%define CONTEXT_OFFSET VARIABLE_OFFSET + +;; Full stack frame layout: +;; RETURN ADDRESS + ARGS +;; R14 = + 16*6 -> --------------------------- +;; GCM CONTEXT (JOB API only) +;; + 6*8 -> --------------------------- +;; GP STORAGE +;; + 16*10 -> -------------------------- +;; XMM STORAGE (windows only) +;; + 16*7 -> -------------------------- +;; LOCAL STORAGE +;; RSP = -> -------------------------- ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; Utility Macros @@ -512,7 +542,7 @@ je %%_CALC_AAD_done movdqu %%XTMP0, [%%GDATA_KEY + HashKey] - READ_SMALL_DATA_INPUT_SSE %%XTMP1, %%T1, %%T2, %%T3, %%T4, %%T5 + READ_SMALL_DATA_INPUT_SSE %%XTMP1, %%T1, %%T2, %%T3 ;byte-reflect the AAD data pshufb %%XTMP1, [rel SHUF_MASK] pxor %%AAD_HASH, %%XTMP1 @@ -551,7 +581,7 @@ %%_fewer_than_16_bytes: lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - READ_SMALL_DATA_INPUT_SSE xmm1, r10, %%PLAIN_CYPH_LEN, rax, r12, r15 + READ_SMALL_DATA_INPUT_SSE xmm1, r10, %%PLAIN_CYPH_LEN, rax %%_data_read: ;Finished reading in data @@ -714,16 +744,18 @@ %assign i (i+1) %endrep -movdqu %%T_key, [%%GDATA_KEY+16*0] + ;; ARK + movdqu %%T_key, [%%GDATA_KEY+16*0] %assign i (9-%%num_initial_blocks) %rep %%num_initial_blocks pxor reg(i),%%T_key %assign i (i+1) %endrep + ;; encrypt N blocks (13 rounds for AES-256, 11 for AES-192 and 9 for AES-128) %assign j 1 -%rep NROUNDS ; encrypt N blocks with 13 key rounds (11 for GCM192) -movdqu %%T_key, [%%GDATA_KEY+16*j] +%rep NROUNDS + movdqu %%T_key, [%%GDATA_KEY+16*j] %assign i (9-%%num_initial_blocks) %rep %%num_initial_blocks aesenc reg(i),%%T_key @@ -732,27 +764,58 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] %assign j (j+1) %endrep - -movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 for GCM192) + ;; last encryption round (14th for AES-256, 12th for AES-192 and 10th for AES-128) + movdqu %%T_key, [%%GDATA_KEY+16*j] %assign i (9-%%num_initial_blocks) %rep %%num_initial_blocks - aesenclast reg(i),%%T_key + aesenclast reg(i), %%T_key %assign i (i+1) %endrep -%assign i (9-%%num_initial_blocks) + ;; - load source buffer block + ;; - XOR AES result with plain/cipher text block + ;; - store to destination is postponed for later +%assign i (9 - %%num_initial_blocks) +%assign k 0 %rep %%num_initial_blocks - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + k*16] pxor reg(i), %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) ; write back ciphertext for %%num_initial_blocks blocks - add %%DATA_OFFSET, 16 - %ifidn %%ENC_DEC, DEC - movdqa reg(i), %%T1 - %endif - pshufb reg(i), [SHUF_MASK] ; prepare ciphertext for GHASH computations -%assign i (i+1) +%ifidn %%ENC_DEC, DEC + ;; DECRYPT +%if k == 0 + movdqa %%T6, %%T1 ; save cipher text block 0 in a register +%else + movdqa [rsp + TMP %+ i], %%T1 ; aligned store of cipher text into stack +%endif +%endif + +%assign i (i + 1) +%assign k (k + 1) %endrep + ;; Note: Loads done first and then stores. + ;; This is to avoid potential loads being blocked on + ;; false store-to-load condition. +%assign i (9 - %%num_initial_blocks) +%assign k 0 +%rep %%num_initial_blocks + XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + k*16], reg(i) +%ifidn %%ENC_DEC, DEC + ;; DECRYPT + ;; - restore cipher text blocks into XMM1:XMM8 +%if k == 0 + movdqa reg(i), %%T6 +%else + movdqa reg(i), [rsp + TMP %+ i] +%endif +%endif + pshufb reg(i), [rel SHUF_MASK] ; prepare ciphertext for GHASH computations +%assign i (i + 1) +%assign k (k + 1) +%endrep + + add %%DATA_OFFSET, (16 * %%num_initial_blocks) + %assign i (8-%%num_initial_blocks) %assign j (9-%%num_initial_blocks) @@ -830,82 +893,58 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 %endrep movdqu %%T_key, [%%GDATA_KEY+16*i] ; do final key round - aesenclast %%XMM1, %%T_key - aesenclast %%XMM2, %%T_key - aesenclast %%XMM3, %%T_key - aesenclast %%XMM4, %%T_key - aesenclast %%XMM5, %%T_key - aesenclast %%XMM6, %%T_key - aesenclast %%XMM7, %%T_key - aesenclast %%XMM8, %%T_key - - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*0] - pxor %%XMM1, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*0], %%XMM1 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM1, %%T1 - %endif - - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*1] - pxor %%XMM2, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*1], %%XMM2 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM2, %%T1 - %endif - - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*2] - pxor %%XMM3, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*2], %%XMM3 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM3, %%T1 - %endif - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*3] - pxor %%XMM4, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*3], %%XMM4 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM4, %%T1 - %endif + ;; - load source buffer block + ;; - do the last AES encryption round + ;; - XOR AES result with plain/cipher text block + ;; - store to destination is postponed for later +%assign i 0 +%assign j 1 +%rep 8 + XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*i] + aesenclast reg(j), %%T_key ; XMM1:XMM8 + pxor reg(j), %%T1 +%ifidn %%ENC_DEC, DEC + ;; DECRYPT +%if j == 1 + movdqa %%T6, %%T1 ; save cipher text block 0 in a register +%else + movdqa [rsp + TMP %+ j], %%T1 ; aligned store of cipher text into stack +%endif +%endif - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*4] - pxor %%XMM5, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*4], %%XMM5 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM5, %%T1 - %endif +%assign i (i+1) +%assign j (j+1) +%endrep - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*5] - pxor %%XMM6, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*5], %%XMM6 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM6, %%T1 - %endif + ;; Note: Loads done first and then stores. + ;; This is to avoid potential loads being blocked on + ;; false store-to-load condition. +%assign i 0 +%assign j 1 +%rep 8 + ;; store cipher/plain text block + XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*i], reg(j) - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*6] - pxor %%XMM7, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*6], %%XMM7 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM7, %%T1 - %endif +%ifidn %%ENC_DEC, DEC + ;; DECRYPT + ;; - restore cipher text blocks into XMM1:XMM8 +%if j == 1 + movdqa reg(j), %%T6 +%else + movdqa reg(j), [rsp + TMP %+ j] +%endif +%endif + ;; prepare cipher text block for GHASH + pshufb reg(j), [rel SHUF_MASK] ; perform a 16Byte swap - XLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*7] - pxor %%XMM8, %%T1 - XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*7], %%XMM8 - %ifidn %%ENC_DEC, DEC - movdqa %%XMM8, %%T1 - %endif +%assign i (i+1) +%assign j (j+1) +%endrep add %%DATA_OFFSET, 128 - pshufb %%XMM1, [SHUF_MASK] ; perform a 16Byte swap pxor %%XMM1, %%T3 ; combine GHASHed value with the corresponding ciphertext - pshufb %%XMM2, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM3, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM4, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM5, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM6, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM7, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM8, [SHUF_MASK] ; perform a 16Byte swap ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1221,6 +1260,17 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 aesenc %%XMM7, %%T1 aesenc %%XMM8, %%T1 + ;; accumulate product into %%T4:%%T7 + pxor %%T2, %%T6 + pxor %%T2, %%T4 + pxor %%T2, %%T7 + + movdqa %%T3, %%T2 + pslldq %%T3, 8 ; shift-L %%T3 2 DWs + psrldq %%T2, 8 ; shift-R %%T2 2 DWs + pxor %%T7, %%T3 + pxor %%T4, %%T2 ; accumulate the results in %%T4:%%T7 + %ifdef GCM128_MODE movdqu %%T5, [%%GDATA + 16*10] %endif @@ -1291,37 +1341,30 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 movdqu %%T5, [%%GDATA + 16*14] ; finish last key round %endif + ;; - load source buffer block + ;; - do the last AES encryption round + ;; - XOR AES result with plain/cipher text block + ;; - store to destination is postponed for later %assign i 0 %assign j 1 %rep 8 - XLDR %%T1, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] - + XLDR %%T1, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + aesenclast reg(j), %%T5 ; XMM1:XMM8 + pxor reg(j), %%T1 %ifidn %%ENC_DEC, DEC - movdqa %%T3, %%T1 + ;; DECRYPT +%if j == 1 + movdqa %%T6, %%T1 ; save cipher text block 0 in a register +%else + movdqa [rsp + TMP %+ j], %%T1 ; aligned store of cipher text into stack %endif - - pxor %%T1, %%T5 - aesenclast reg(j), %%T1 ; XMM1:XMM8 - XSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], reg(j) ; Write to the Output buffer - -%ifidn %%ENC_DEC, DEC - movdqa reg(j), %%T3 %endif + %assign i (i+1) %assign j (j+1) %endrep - pxor %%T2, %%T6 - pxor %%T2, %%T4 - pxor %%T2, %%T7 - - movdqa %%T3, %%T2 - pslldq %%T3, 8 ; shift-L %%T3 2 DWs - psrldq %%T2, 8 ; shift-R %%T2 2 DWs - pxor %%T7, %%T3 - pxor %%T4, %%T2 ; accumulate the results in %%T4:%%T7 - - ;first phase of the reduction + ;; first phase of the reduction movdqa %%T2, %%T7 movdqa %%T3, %%T7 movdqa %%T1, %%T7 ; move %%T7 into %%T2, %%T3, %%T1 in order to perform the three shifts independently @@ -1337,18 +1380,35 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 pslldq %%T2, 12 ; shift-L %%T2 3 DWs pxor %%T7, %%T2 ; first phase of the reduction complete - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - pshufb %%XMM1, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM2, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM3, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM4, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM5, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM6, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM7, [SHUF_MASK] ; perform a 16Byte swap - pshufb %%XMM8, [SHUF_MASK] ; perform a 16Byte swap + ;; Note: Loads done first and then stores. + ;; This is to avoid potential loads being blocked on + ;; false store-to-load condition. +%assign i 0 +%assign j 1 +%rep 8 + ;; store cipher/plain text block + XSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*i], reg(j) - ;second phase of the reduction +%ifidn %%ENC_DEC, DEC + ;; DECRYPT + ;; - restore cipher text blocks into XMM1:XMM8 +%if j == 1 + movdqa reg(j), %%T6 +%else + movdqa reg(j), [rsp + TMP %+ j] +%endif +%endif + ;; prepare cipher text block for GHASH + pshufb reg(j), [rel SHUF_MASK] ; perform a 16Byte swap + +%assign i (i+1) +%assign j (j+1) +%endrep + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ;; second phase of the reduction movdqa %%T2,%%T7 ; make 3 copies of %%T7 (in in %%T2, %%T3, %%T1) for doing three shift operations movdqa %%T3,%%T7 movdqa %%T1,%%T7 @@ -1567,58 +1627,65 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 ;; Start of Stack Setup -%macro FUNC_SAVE 0 +%macro FUNC_SAVE 0-1 ;; Required for Update/GCM_ENC - ;the number of pushes must equal STACK_OFFSET - push r12 - push r13 - push r14 - push r15 - mov r14, rsp + mov rax, rsp +%if %0 == 0 sub rsp, VARIABLE_OFFSET +%else +%ifidni %1, alloc_context + sub rsp, VARIABLE_OFFSET + CONTEXT_SIZE +%endif +%endif and rsp, ~63 + mov [rsp + GP_OFFSET + 0*8], rax ; original rsp pointer + mov [rsp + GP_OFFSET + 1*8], r12 + mov [rsp + GP_OFFSET + 2*8], r13 + mov [rsp + GP_OFFSET + 3*8], r14 + mov [rsp + GP_OFFSET + 4*8], r15 + mov r14, rax + %ifidn __OUTPUT_FORMAT__, win64 ; xmm6:xmm15 need to be maintained for Windows - movdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 - movdqu [rsp + LOCAL_STORAGE + 1*16],xmm7 - movdqu [rsp + LOCAL_STORAGE + 2*16],xmm8 - movdqu [rsp + LOCAL_STORAGE + 3*16],xmm9 - movdqu [rsp + LOCAL_STORAGE + 4*16],xmm10 - movdqu [rsp + LOCAL_STORAGE + 5*16],xmm11 - movdqu [rsp + LOCAL_STORAGE + 6*16],xmm12 - movdqu [rsp + LOCAL_STORAGE + 7*16],xmm13 - movdqu [rsp + LOCAL_STORAGE + 8*16],xmm14 - movdqu [rsp + LOCAL_STORAGE + 9*16],xmm15 + movdqu [rsp + LOCAL_STORAGE + 0*16], xmm6 + movdqu [rsp + LOCAL_STORAGE + 1*16], xmm7 + movdqu [rsp + LOCAL_STORAGE + 2*16], xmm8 + movdqu [rsp + LOCAL_STORAGE + 3*16], xmm9 + movdqu [rsp + LOCAL_STORAGE + 4*16], xmm10 + movdqu [rsp + LOCAL_STORAGE + 5*16], xmm11 + movdqu [rsp + LOCAL_STORAGE + 6*16], xmm12 + movdqu [rsp + LOCAL_STORAGE + 7*16], xmm13 + movdqu [rsp + LOCAL_STORAGE + 8*16], xmm14 + movdqu [rsp + LOCAL_STORAGE + 9*16], xmm15 %endif %endmacro %macro FUNC_RESTORE 0 %ifdef SAFE_DATA - clear_scratch_gps_asm clear_scratch_xmms_sse_asm %endif %ifidn __OUTPUT_FORMAT__, win64 - movdqu xmm15 , [rsp + LOCAL_STORAGE + 9*16] - movdqu xmm14 , [rsp + LOCAL_STORAGE + 8*16] - movdqu xmm13 , [rsp + LOCAL_STORAGE + 7*16] - movdqu xmm12 , [rsp + LOCAL_STORAGE + 6*16] - movdqu xmm11 , [rsp + LOCAL_STORAGE + 5*16] - movdqu xmm10 , [rsp + LOCAL_STORAGE + 4*16] - movdqu xmm9 , [rsp + LOCAL_STORAGE + 3*16] - movdqu xmm8 , [rsp + LOCAL_STORAGE + 2*16] - movdqu xmm7 , [rsp + LOCAL_STORAGE + 1*16] - movdqu xmm6 , [rsp + LOCAL_STORAGE + 0*16] + movdqu xmm15, [rsp + LOCAL_STORAGE + 9*16] + movdqu xmm14, [rsp + LOCAL_STORAGE + 8*16] + movdqu xmm13, [rsp + LOCAL_STORAGE + 7*16] + movdqu xmm12, [rsp + LOCAL_STORAGE + 6*16] + movdqu xmm11, [rsp + LOCAL_STORAGE + 5*16] + movdqu xmm10, [rsp + LOCAL_STORAGE + 4*16] + movdqu xmm9, [rsp + LOCAL_STORAGE + 3*16] + movdqu xmm8, [rsp + LOCAL_STORAGE + 2*16] + movdqu xmm7, [rsp + LOCAL_STORAGE + 1*16] + movdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] %endif -;; Required for Update/GCM_ENC - mov rsp, r14 - pop r15 - pop r14 - pop r13 - pop r12 + ;; Required for Update/GCM_ENC + mov r12, [rsp + GP_OFFSET + 1*8] + mov r13, [rsp + GP_OFFSET + 2*8] + mov r14, [rsp + GP_OFFSET + 3*8] + mov r15, [rsp + GP_OFFSET + 4*8] + mov rsp, [rsp + GP_OFFSET + 0*8] %endmacro %macro CALC_J0 15 @@ -1862,7 +1929,7 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 jge %%_large_enough_update lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] - READ_SMALL_DATA_INPUT_SSE xmm1, r10, r13, r12, r15, rax + READ_SMALL_DATA_INPUT_SSE xmm1, r10, r13, r12 lea r12, [SHIFT_MASK + 16] sub r12, r13 jmp %%_data_read @@ -2044,7 +2111,7 @@ movdqu %%T_key, [%%GDATA_KEY+16*j] ; encrypt with last (14th) key round (12 %%_fewer_than_16_bytes: lea r10, [%%PLAIN_IN] - READ_SMALL_DATA_INPUT_SSE xmm1, r10, %%PLAIN_LEN, rax, r12, r15 + READ_SMALL_DATA_INPUT_SSE xmm1, r10, %%PLAIN_LEN, rax ; Finished reading in data %%_data_read: diff --git a/lib/avx512_t2/gcm_vaes_avx512.inc b/lib/include/gcm_vaes_avx512.inc similarity index 76% rename from lib/avx512_t2/gcm_vaes_avx512.inc rename to lib/include/gcm_vaes_avx512.inc index e6241b32259e4196eaea8bf30abcb155984c13e9..8298b341cc78ace90629c88337f0c81ad78676ca 100644 --- a/lib/avx512_t2/gcm_vaes_avx512.inc +++ b/lib/include/gcm_vaes_avx512.inc @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -114,19 +114,19 @@ %ifndef GCM_VAES_AVX512_INC %define GCM_VAES_AVX512_INC -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" -%include "include/gcm_defines.asm" -%include "include/gcm_keys_vaes_avx512.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" +%include "include/gcm_defines.inc" +%include "include/gcm_keys_vaes_avx512.inc" %include "include/gcm_common.inc" -%include "include/memcpy.asm" -%include "include/aes_common.asm" +%include "include/memcpy.inc" +%include "include/aes_common.inc" %include "include/cet.inc" %ifndef GCM128_MODE %ifndef GCM192_MODE %ifndef GCM256_MODE -%error "No GCM mode selected for gcm_avx512.asm!" +%error "No GCM mode selected for gcm_avx512.inc!" %endif %endif %endif @@ -150,6 +150,15 @@ %define GMAC_FN_NAME(x) imb_aes_gmac_ %+ x %+ _256_ %+ vaes_avx512 %endif +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Pipeline parameters +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; reduction every 32 or 16 blocks +%define big_loop_nblocks 32 +;; cipher lead (depth) is 32 blocks +%define big_loop_depth 32 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Stack frame definition ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -160,16 +169,30 @@ %define XMM_STORAGE 0 %define GP_STORAGE (8*8) ; space for 7 GP registers + 1 for alignment %endif -%define LOCAL_STORAGE (48*16) ; space for up to 48 AES blocks +%define LOCAL_STORAGE (big_loop_nblocks*16) ; space for cipher text blocks for GHASH ;;; sequence is (bottom-up): GP, XMM, local %define STACK_GP_OFFSET 0 %define STACK_XMM_OFFSET (STACK_GP_OFFSET + GP_STORAGE) %define STACK_LOCAL_OFFSET (STACK_XMM_OFFSET + XMM_STORAGE) %define STACK_FRAME_SIZE (STACK_LOCAL_OFFSET + LOCAL_STORAGE) - -;; for compatibility with stack argument definitions in gcm_defines.asm -%define STACK_OFFSET 0 +%define STACK_FRAME_SIZE_SMALL (STACK_LOCAL_OFFSET) + +;; extra memory for GCM context structure +%define CONTEXT_SIZE (6*16) +%define CONTEXT_OFFSET STACK_FRAME_SIZE + +;; Full stack frame layout: +;; RETURN ADDRESS + ARGS +;; R14 = + 16*6 -> --------------------------- +;; GCM CONTEXT (JOB API only) +;; + 32*16 -> --------------------------- +;; LOCAL STORAGE +;; + 16*10 -> -------------------------- +;; XMM STORAGE (windows only) +;; + 8*8 -> -------------------------- +;; GP STORAGE +;; RSP = -> -------------------------- ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Utility Macros @@ -197,35 +220,6 @@ vpxorq XWORD(%%REG), XWORD(%%REG), XWORD(%%TMP) %endmacro ; VHPXORI2x128 -;;; =========================================================================== -;;; =========================================================================== -;;; AVX512 reduction macro -%macro VCLMUL_REDUCE 6 -%define %%OUT %1 ; [out] zmm/ymm/xmm: result (must not be %%TMP1 or %%HI128) -%define %%POLY %2 ; [in] zmm/ymm/xmm: polynomial -%define %%HI128 %3 ; [in] zmm/ymm/xmm: high 128b of hash to reduce -%define %%LO128 %4 ; [in] zmm/ymm/xmm: low 128b of hash to reduce -%define %%TMP0 %5 ; [in] zmm/ymm/xmm: temporary register -%define %%TMP1 %6 ; [in] zmm/ymm/xmm: temporary register - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;; first phase of the reduction - vpclmulqdq %%TMP0, %%POLY, %%LO128, 0x01 - vpslldq %%TMP0, %%TMP0, 8 ; shift-L 2 DWs - vpxorq %%TMP0, %%LO128, %%TMP0 ; first phase of the reduction complete - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - ;; second phase of the reduction - vpclmulqdq %%TMP1, %%POLY, %%TMP0, 0x00 - vpsrldq %%TMP1, %%TMP1, 4 ; shift-R only 1-DW to obtain 2-DWs shift-R - - vpclmulqdq %%OUT, %%POLY, %%TMP0, 0x10 - vpslldq %%OUT, %%OUT, 4 ; shift-L 1-DW to obtain result with no shifts - - vpternlogq %%OUT, %%TMP1, %%HI128, 0x96 ; OUT/GHASH = OUT xor TMP1 xor HI128 - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%endmacro - ;;; =========================================================================== ;;; =========================================================================== ;;; schoolbook multiply of 16 blocks (8 x 16 bytes) @@ -233,11 +227,11 @@ ;;; %%INPTR address is 64 byte aligned ;;; - there is an option to pass ready blocks through ZMM registers too. ;;; 4 extra parameters need to passed in such case and 21st argument can be empty -%macro GHASH_16 21-22 +%macro GHASH_16 22-23 %define %%TYPE %1 ; [in] ghash type: start (xor hash), mid, end (same as mid; no reduction), - ; end_reduce (end with reduction), start_reduce -%define %%GH %2 ; [in/out] ZMM ghash sum: high 128-bits -%define %%GM %3 ; [in/out] ZMM ghash sum: middle 128-bits + ; end_reduce (end with reduction), start_reduce, end_reduce_no_hxor +%define %%LOADT %2 ; [in] hash key load type: hk_load or hk_bcast +%define %%GH %3 ; [in/out] ZMM ghash sum: high 128-bits %define %%GL %4 ; [in/out] ZMM ghash sum: low 128-bits %define %%INPTR %5 ; [in] data input pointer %define %%INOFF %6 ; [in] data input offset @@ -254,12 +248,24 @@ %define %%ZTMP5 %17 ; [clobbered] temporary ZMM %define %%ZTMP6 %18 ; [clobbered] temporary ZMM %define %%ZTMP7 %19 ; [clobbered] temporary ZMM -%define %%ZTMP8 %20 ; [clobbered] temporary ZMM -%define %%ZTMP9 %21 ; [clobbered] temporary ZMM -%define %%SHUFM %22 ; [in] ZMM with shuffle mask - provided only when input data needs shuffling +%define %%ZTMP8 %20 ; [clobbered**] temporary ZMM +%define %%ZTMP9 %21 ; [clobbered**] temporary ZMM +%define %%ZTMPA %22 ; [clobbered**] temporary ZMM +%define %%SHUFM %23 ; [in] ZMM with shuffle mask - provided only when input data needs shuffling %assign start_ghash 0 %assign do_reduction 0 +%assign uload_and_shuffle 0 +%assign hk_broadcast 0 +%assign do_hxor 1 + +%if %0 == 23 +%assign uload_and_shuffle 1 +%endif + +%ifidn %%LOADT, hk_bcast +%assign hk_broadcast 1 +%endif %ifidn %%TYPE, start %assign start_ghash 1 @@ -274,8 +280,12 @@ %assign do_reduction 1 %endif +%ifidn %%TYPE, end_reduce_no_hxor +%assign do_reduction 1 +%assign do_hxor 0 +%endif ;; ghash blocks 0-3 -%if %0 == 22 +%if uload_and_shuffle != 0 vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS] vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else @@ -285,80 +295,97 @@ %if start_ghash != 0 vpxorq %%ZTMP9, %%ZTMP9, %%HASH %endif +%if hk_broadcast != 0 + vbroadcastf64x2 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS] + vbroadcastf64x2 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap] +%else vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS] - vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMP8, 0x11 ; T0H = a1*b1 - vpclmulqdq %%ZTMP1, %%ZTMP9, %%ZTMP8, 0x00 ; T0L = a0*b0 - vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; T0M1 = a1*b0 - vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x10 ; T0M2 = a0*b1 + vmovdqu64 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap] +%endif + vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMPA, 0x00 ; TLL = ML*KL + vpclmulqdq %%ZTMP1, %%ZTMP9, %%ZTMPA, 0x10 ; TLH = ML*KH + vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; THL = MH*HL + vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x11 ; THH = MH*HH ;; ghash blocks 4-7 -%if %0 == 22 +%if uload_and_shuffle != 0 vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 64] vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 64] %endif +%if hk_broadcast != 0 + vbroadcastf64x2 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 64] + vbroadcastf64x2 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap + 64] +%else vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 64] - vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMP8, 0x11 ; T1H = a1*b1 - vpclmulqdq %%ZTMP5, %%ZTMP9, %%ZTMP8, 0x00 ; T1L = a0*b0 - vpclmulqdq %%ZTMP6, %%ZTMP9, %%ZTMP8, 0x01 ; T1M1 = a1*b0 - vpclmulqdq %%ZTMP7, %%ZTMP9, %%ZTMP8, 0x10 ; T1M2 = a0*b1 + vmovdqu64 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap + 64] +%endif + vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMPA, 0x00 ; TLL = ML*KL + vpclmulqdq %%ZTMP5, %%ZTMP9, %%ZTMPA, 0x10 ; TLH = ML*KH + vpclmulqdq %%ZTMP6, %%ZTMP9, %%ZTMP8, 0x01 ; THL = MH*HL + vpclmulqdq %%ZTMP7, %%ZTMP9, %%ZTMP8, 0x11 ; THH = MH*HH ;; update sums %if start_ghash != 0 - vpxorq %%GM, %%ZTMP2, %%ZTMP6 ; GM = T0M1 + T1M1 - vpxorq %%GH, %%ZTMP0, %%ZTMP4 ; GH = T0H + T1H - vpxorq %%GL, %%ZTMP1, %%ZTMP5 ; GL = T0L + T1L - vpternlogq %%GM, %%ZTMP3, %%ZTMP7, 0x96 ; GM = T0M2 + T1M1 + vpxorq %%GL, %%ZTMP0, %%ZTMP2 ; T2 = THL + TLL + vpxorq %%GH, %%ZTMP1, %%ZTMP3 ; T1 = THH + TLH %else ;; mid, end, end_reduce - vpternlogq %%GM, %%ZTMP2, %%ZTMP6, 0x96 ; GM += T0M1 + T1M1 - vpternlogq %%GH, %%ZTMP0, %%ZTMP4, 0x96 ; GH += T0H + T1H - vpternlogq %%GL, %%ZTMP1, %%ZTMP5, 0x96 ; GL += T0L + T1L - vpternlogq %%GM, %%ZTMP3, %%ZTMP7, 0x96 ; GM += T0M2 + T1M1 + vpternlogq %%GL, %%ZTMP0, %%ZTMP2, 0x96 ; T2 = THL + TLL + vpternlogq %%GH, %%ZTMP1, %%ZTMP3, 0x96 ; T1 = THH + TLH %endif ;; ghash blocks 8-11 -%if %0 == 22 +%if uload_and_shuffle != 0 vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 128] vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 128] %endif +%if hk_broadcast != 0 + vbroadcastf64x2 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 128] + vbroadcastf64x2 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap + 128] +%else vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 128] - vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMP8, 0x11 ; T0H = a1*b1 - vpclmulqdq %%ZTMP1, %%ZTMP9, %%ZTMP8, 0x00 ; T0L = a0*b0 - vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; T0M1 = a1*b0 - vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x10 ; T0M2 = a0*b1 + vmovdqu64 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap + 128] +%endif + vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMPA, 0x00 ; TLL = ML*KL + vpclmulqdq %%ZTMP1, %%ZTMP9, %%ZTMPA, 0x10 ; TLH = ML*KH + vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; THL = MH*HL + vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x11 ; THH = MH*HH + ;; update sums + vpternlogq %%GL, %%ZTMP6, %%ZTMP4, 0x96 ; T2 = THL + TLL + vpternlogq %%GH, %%ZTMP7, %%ZTMP5, 0x96 ; T1 = THH + TLH ;; ghash blocks 12-15 -%if %0 == 22 +%if uload_and_shuffle != 0 vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 192] vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 192] %endif +%if hk_broadcast != 0 + vbroadcastf64x2 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 192] + vbroadcastf64x2 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap + 192] +%else vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 192] - vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMP8, 0x11 ; T1H = a1*b1 - vpclmulqdq %%ZTMP5, %%ZTMP9, %%ZTMP8, 0x00 ; T1L = a0*b0 - vpclmulqdq %%ZTMP6, %%ZTMP9, %%ZTMP8, 0x01 ; T1M1 = a1*b0 - vpclmulqdq %%ZTMP7, %%ZTMP9, %%ZTMP8, 0x10 ; T1M2 = a0*b1 + vmovdqu64 %%ZTMPA, [%%HKPTR + %%HKOFF + %%HKDIS + HKeyGap + 192] +%endif + vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMPA, 0x00 ; TLL = ML*KL + vpclmulqdq %%ZTMP5, %%ZTMP9, %%ZTMPA, 0x10 ; TLH = ML*KH + vpclmulqdq %%ZTMP6, %%ZTMP9, %%ZTMP8, 0x01 ; THL = MH*HL + vpclmulqdq %%ZTMP7, %%ZTMP9, %%ZTMP8, 0x11 ; THH = MH*HH ;; update sums - vpternlogq %%GM, %%ZTMP2, %%ZTMP6, 0x96 ; GM += T0M1 + T1M1 - vpternlogq %%GH, %%ZTMP0, %%ZTMP4, 0x96 ; GH += T0H + T1H - vpternlogq %%GL, %%ZTMP1, %%ZTMP5, 0x96 ; GL += T0L + T1L - vpternlogq %%GM, %%ZTMP3, %%ZTMP7, 0x96 ; GM += T0M2 + T1M1 + vpternlogq %%GL, %%ZTMP0, %%ZTMP2, 0x96 ; T2 = THL + TLL + vpternlogq %%GH, %%ZTMP1, %%ZTMP3, 0x96 ; T1 = THH + TLH + vpternlogq %%GL, %%ZTMP6, %%ZTMP4, 0x96 ; T2 = THL + TLL + vpternlogq %%GH, %%ZTMP7, %%ZTMP5, 0x96 ; T1 = THH + TLH + ;; **ZTMP8 and ZTMPA include hash keys %if do_reduction != 0 - ;; integrate GM into GH and GL - vpsrldq %%ZTMP0, %%GM, 8 - vpslldq %%ZTMP1, %%GM, 8 - vpxorq %%GH, %%GH, %%ZTMP0 - vpxorq %%GL, %%GL, %%ZTMP1 - - ;; add GH and GL 128-bit words horizontally - VHPXORI4x128 %%GH, %%ZTMP0 - VHPXORI4x128 %%GL, %%ZTMP1 - - ;; reduction - vmovdqa64 XWORD(%%ZTMP2), [rel POLY2] - VCLMUL_REDUCE XWORD(%%HASH), XWORD(%%ZTMP2), \ - XWORD(%%GH), XWORD(%%GL), XWORD(%%ZTMP0), XWORD(%%ZTMP1) + ;; new reduction + vpclmulqdq %%HASH, %%GL, [rel POLY], 0x10 + vpshufd %%ZTMP0, %%GL, 01001110b + vpternlogq %%HASH, %%GH, %%ZTMP0, 0x96 +%if do_hxor != 0 + VHPXORI4x128 %%HASH, %%ZTMP0 +%endif %endif %endmacro @@ -369,112 +396,141 @@ ;;; - it doesn't load the data and it assumed it is already loaded and ;;; shuffled ;;; - single_call scenario only -%macro GHASH_1_TO_16 17-20 +%macro GHASH_1_TO_16 18-20 %define %%KP %1 ; [in] pointer to expanded keys %define %%GHASH %2 ; [out] ghash output -%define %%T0H %3 ; [clobbered] temporary ZMM -%define %%T0L %4 ; [clobbered] temporary ZMM -%define %%T0M1 %5 ; [clobbered] temporary ZMM -%define %%T0M2 %6 ; [clobbered] temporary ZMM -%define %%T1H %7 ; [clobbered] temporary ZMM -%define %%T1L %8 ; [clobbered] temporary ZMM -%define %%T1M1 %9 ; [clobbered] temporary ZMM -%define %%T1M2 %10 ; [clobbered] temporary ZMM -%define %%HK %11 ; [clobbered] temporary ZMM -%define %%AAD_HASH_IN %12 ; [in] input hash value -%define %%CIPHER_IN0 %13 ; [in] ZMM with cipher text blocks 0-3 -%define %%CIPHER_IN1 %14 ; [in] ZMM with cipher text blocks 4-7 -%define %%CIPHER_IN2 %15 ; [in] ZMM with cipher text blocks 8-11 -%define %%CIPHER_IN3 %16 ; [in] ZMM with cipher text blocks 12-15 -%define %%NUM_BLOCKS %17 ; [in] numerical value, number of blocks -%define %%GH %18 ; [in] ZMM with hi product part -%define %%GM %19 ; [in] ZMM with mid product part +%define %%THH1 %3 ; [clobbered] temporary ZMM +%define %%THL1 %4 ; [clobbered] temporary ZMM +%define %%TLH1 %5 ; [clobbered] temporary ZMM +%define %%TLL1 %6 ; [clobbered] temporary ZMM +%define %%THH2 %7 ; [clobbered] temporary ZMM +%define %%THL2 %8 ; [clobbered] temporary ZMM +%define %%TLH2 %9 ; [clobbered] temporary ZMM +%define %%TLL2 %10 ; [clobbered] temporary ZMM +%define %%HK1 %11 ; [clobbered] temporary ZMM +%define %%HK2 %12 ; [clobbered] temporary ZMM +%define %%AAD_HASH_IN %13 ; [in] input hash value +%define %%CIPHER_IN0 %14 ; [in**] ZMM with cipher text blocks 0-3 +%define %%CIPHER_IN1 %15 ; [in**] ZMM with cipher text blocks 4-7 +%define %%CIPHER_IN2 %16 ; [in**] ZMM with cipher text blocks 8-11 +%define %%CIPHER_IN3 %17 ; [in**] ZMM with cipher text blocks 12-15 +%define %%NUM_BLOCKS %18 ; [in] numerical value, number of blocks +%define %%GH %19 ; [in] ZMM with hi product part %define %%GL %20 ; [in] ZMM with lo product part %assign hashk HashKey_ %+ %%NUM_BLOCKS -%if %0 == 17 +%if %0 == 18 vpxorq %%CIPHER_IN0, %%CIPHER_IN0, %%AAD_HASH_IN %endif %if %%NUM_BLOCKS == 16 - - vmovdqu64 %%HK, [%%KP + hashk] - vpclmulqdq %%T0H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T0L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T0M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T0M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 - vmovdqu64 %%HK, [%%KP + hashk + (1*64)] - vpclmulqdq %%T1H, %%CIPHER_IN1, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T1L, %%CIPHER_IN1, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T1M1, %%CIPHER_IN1, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T1M2, %%CIPHER_IN1, %%HK, 0x10 ; M2 = a0*b1 - vmovdqu64 %%HK, [%%KP + hashk + (2*64)] - vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x00 ; L = a0*b0 - vpternlogq %%T0H, %%CIPHER_IN0, %%T1H, 0x96 - vpternlogq %%T0L, %%CIPHER_IN1, %%T1L, 0x96 - vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x10 ; M2 = a0*b1 - vpternlogq %%T0M1, %%CIPHER_IN0, %%T1M1, 0x96 - vpternlogq %%T0M2, %%CIPHER_IN1, %%T1M2, 0x96 - vmovdqu64 %%HK, [%%KP + hashk + (3*64)] - vpclmulqdq %%T1H, %%CIPHER_IN3, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T1L, %%CIPHER_IN3, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T1M1, %%CIPHER_IN3, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T1M2, %%CIPHER_IN3, %%HK, 0x10 ; M2 = a0*b1 - vpxorq %%T1H, %%T0H, %%T1H - vpxorq %%T1L, %%T0L, %%T1L - vpxorq %%T1M1, %%T0M1, %%T1M1 - vpxorq %%T1M2, %%T0M2, %%T1M2 + vmovdqu64 %%HK1, [%%KP + hashk] + vmovdqu64 %%HK2, [%%KP + hashk + HKeyGap] + vpclmulqdq %%TLL1, %%CIPHER_IN0, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH1, %%CIPHER_IN0, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL1, %%CIPHER_IN0, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH1, %%CIPHER_IN0, %%HK1, 0x11 ; THH = MH*HH + + vmovdqu64 %%HK1, [%%KP + hashk + (1*64)] + vmovdqu64 %%HK2, [%%KP + hashk + (1*64) + HKeyGap] + vpclmulqdq %%TLL2, %%CIPHER_IN1, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%CIPHER_IN1, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%CIPHER_IN1, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%CIPHER_IN1, %%HK1, 0x11 ; THH = MH*HH + + vmovdqu64 %%HK1, [%%KP + hashk + (2*64)] + vmovdqu64 %%HK2, [%%KP + hashk + (2*64) + HKeyGap] + vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%HK2, %%CIPHER_IN2, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%HK1, %%CIPHER_IN2, %%HK1, 0x11 ; THH = MH*HH + + ;; add sums so far + vpternlogq %%TLL1, %%TLL2, %%CIPHER_IN0, 0x96 + vpternlogq %%TLH1, %%TLH2, %%CIPHER_IN1, 0x96 + vpternlogq %%THL1, %%THL2, %%HK2, 0x96 + vpternlogq %%THH1, %%THH2, %%HK1, 0x96 + + ;; the last multiply + vmovdqu64 %%HK1, [%%KP + hashk + (3*64)] + vmovdqu64 %%HK2, [%%KP + hashk + (3*64) + HKeyGap] + vpclmulqdq %%TLL2, %%CIPHER_IN3, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%CIPHER_IN3, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%CIPHER_IN3, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%CIPHER_IN3, %%HK1, 0x11 ; THH = MH*HH + + ;; get all sums into THH1:TLL1 + vpxorq %%TLL1, %%TLL1, %%THL1 + vpxorq %%THH1, %%THH1, %%TLH1 + vpternlogq %%TLL1, %%TLL2, %%THL2, 0x96 + vpternlogq %%THH1, %%THH2, %%TLH2, 0x96 %elif %%NUM_BLOCKS >= 12 - vmovdqu64 %%HK, [%%KP + hashk] - vpclmulqdq %%T0H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T0L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T0M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T0M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 - vmovdqu64 %%HK, [%%KP + hashk + (1*64)] - vpclmulqdq %%T1H, %%CIPHER_IN1, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T1L, %%CIPHER_IN1, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T1M1, %%CIPHER_IN1, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T1M2, %%CIPHER_IN1, %%HK, 0x10 ; M2 = a0*b1 - vmovdqu64 %%HK, [%%KP + hashk + (2*64)] - vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x00 ; L = a0*b0 - vpternlogq %%T1H, %%CIPHER_IN0, %%T0H, 0x96 - vpternlogq %%T1L, %%CIPHER_IN1, %%T0L, 0x96 - vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x10 ; M2 = a0*b1 - vpternlogq %%T1M1, %%CIPHER_IN0, %%T0M1, 0x96 - vpternlogq %%T1M2, %%CIPHER_IN1, %%T0M2, 0x96 + vmovdqu64 %%HK1, [%%KP + hashk] + vmovdqu64 %%HK2, [%%KP + hashk + HKeyGap] + vpclmulqdq %%TLL1, %%CIPHER_IN0, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH1, %%CIPHER_IN0, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL1, %%CIPHER_IN0, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH1, %%CIPHER_IN0, %%HK1, 0x11 ; THH = MH*HH + + vmovdqu64 %%HK1, [%%KP + hashk + (1*64)] + vmovdqu64 %%HK2, [%%KP + hashk + (1*64) + HKeyGap] + vpclmulqdq %%TLL2, %%CIPHER_IN1, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%CIPHER_IN1, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%CIPHER_IN1, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%CIPHER_IN1, %%HK1, 0x11 ; THH = MH*HH + + vmovdqu64 %%HK1, [%%KP + hashk + (2*64)] + vmovdqu64 %%HK2, [%%KP + hashk + (2*64) + HKeyGap] + vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%HK2, %%CIPHER_IN2, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%HK1, %%CIPHER_IN2, %%HK1, 0x11 ; THH = MH*HH + + ;; add sums into THH1:TLL1 + vpternlogq %%TLL1, %%TLL2, %%CIPHER_IN0, 0x96 + vpternlogq %%TLH1, %%TLH2, %%CIPHER_IN1, 0x96 + vpternlogq %%THL1, %%THL2, %%HK2, 0x96 + vpternlogq %%THH1, %%THH2, %%HK1, 0x96 + vpxorq %%TLL1, %%TLL1, %%THL1 + vpxorq %%THH1, %%THH1, %%TLH1 %elif %%NUM_BLOCKS >= 8 - vmovdqu64 %%HK, [%%KP + hashk] - vpclmulqdq %%T0H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T0L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T0M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T0M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 - vmovdqu64 %%HK, [%%KP + hashk + (1*64)] - vpclmulqdq %%T1H, %%CIPHER_IN1, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T1L, %%CIPHER_IN1, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T1M1, %%CIPHER_IN1, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T1M2, %%CIPHER_IN1, %%HK, 0x10 ; M2 = a0*b1 - vpxorq %%T1H, %%T0H, %%T1H - vpxorq %%T1L, %%T0L, %%T1L - vpxorq %%T1M1, %%T0M1, %%T1M1 - vpxorq %%T1M2, %%T0M2, %%T1M2 + vmovdqu64 %%HK1, [%%KP + hashk] + vmovdqu64 %%HK2, [%%KP + hashk + HKeyGap] + vpclmulqdq %%TLL1, %%CIPHER_IN0, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH1, %%CIPHER_IN0, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL1, %%CIPHER_IN0, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH1, %%CIPHER_IN0, %%HK1, 0x11 ; THH = MH*HH + + vmovdqu64 %%HK1, [%%KP + hashk + (1*64)] + vmovdqu64 %%HK2, [%%KP + hashk + (1*64) + HKeyGap] + vpclmulqdq %%TLL2, %%CIPHER_IN1, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%CIPHER_IN1, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%CIPHER_IN1, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%CIPHER_IN1, %%HK1, 0x11 ; THH = MH*HH + + ;; add sums into THH1:TLL1 + vpxorq %%TLL1, %%TLL1, %%THL1 + vpxorq %%THH1, %%THH1, %%TLH1 + vpternlogq %%TLL1, %%TLL2, %%THL2, 0x96 + vpternlogq %%THH1, %%THH2, %%TLH2, 0x96 %elif %%NUM_BLOCKS >= 4 - vmovdqu64 %%HK, [%%KP + hashk] - vpclmulqdq %%T1H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T1L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 - vpclmulqdq %%T1M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T1M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 + vmovdqu64 %%HK1, [%%KP + hashk] + vmovdqu64 %%HK2, [%%KP + hashk + HKeyGap] + vpclmulqdq %%TLL1, %%CIPHER_IN0, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH1, %%CIPHER_IN0, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL1, %%CIPHER_IN0, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH1, %%CIPHER_IN0, %%HK1, 0x11 ; THH = MH*HH + ;; add sums into THH1:TLL1 + vpxorq %%TLL1, %%TLL1, %%THL1 + vpxorq %%THH1, %%THH1, %%TLH1 %endif ;; T1H/L/M1/M2 - hold current product sums (provided %%NUM_BLOCKS >= 4) @@ -492,100 +548,70 @@ %xdefine %%REG_IN %%CIPHER_IN %+ reg_idx %if blocks_left == 1 - vmovdqu64 XWORD(%%HK), [%%KP + hashk] - vpclmulqdq XWORD(%%T0M1), XWORD(%%REG_IN), XWORD(%%HK), 0x01 ; M1 = a1*b0 - vpclmulqdq XWORD(%%T0M2), XWORD(%%REG_IN), XWORD(%%HK), 0x10 ; M2 = a0*b1 - vpclmulqdq XWORD(%%T0H), XWORD(%%REG_IN), XWORD(%%HK), 0x11 ; H = a1*b1 - vpclmulqdq XWORD(%%T0L), XWORD(%%REG_IN), XWORD(%%HK), 0x00 ; L = a0*b0 + vmovdqu64 XWORD(%%HK1), [%%KP + hashk] + vmovdqu64 XWORD(%%HK2), [%%KP + hashk + HKeyGap] + vpclmulqdq XWORD(%%TLL2), XWORD(%%REG_IN), XWORD(%%HK2), 0x00 ; TLL = ML*KL + vpclmulqdq XWORD(%%TLH2), XWORD(%%REG_IN), XWORD(%%HK2), 0x10 ; TLH = ML*KH + vpclmulqdq XWORD(%%THL2), XWORD(%%REG_IN), XWORD(%%HK1), 0x01 ; THL = MH*HL + vpclmulqdq XWORD(%%THH2), XWORD(%%REG_IN), XWORD(%%HK1), 0x11 ; THH = MH*HH %elif blocks_left == 2 - vmovdqu64 YWORD(%%HK), [%%KP + hashk] - vpclmulqdq YWORD(%%T0M1), YWORD(%%REG_IN), YWORD(%%HK), 0x01 ; M1 = a1*b0 - vpclmulqdq YWORD(%%T0M2), YWORD(%%REG_IN), YWORD(%%HK), 0x10 ; M2 = a0*b1 - vpclmulqdq YWORD(%%T0H), YWORD(%%REG_IN), YWORD(%%HK), 0x11 ; H = a1*b1 - vpclmulqdq YWORD(%%T0L), YWORD(%%REG_IN), YWORD(%%HK), 0x00 ; L = a0*b0 + vmovdqu64 YWORD(%%HK1), [%%KP + hashk] + vmovdqu64 YWORD(%%HK2), [%%KP + hashk + HKeyGap] + vpclmulqdq YWORD(%%TLL2), YWORD(%%REG_IN), YWORD(%%HK2), 0x00 ; TLL = ML*KL + vpclmulqdq YWORD(%%TLH2), YWORD(%%REG_IN), YWORD(%%HK2), 0x10 ; TLH = ML*KH + vpclmulqdq YWORD(%%THL2), YWORD(%%REG_IN), YWORD(%%HK1), 0x01 ; THL = MH*HL + vpclmulqdq YWORD(%%THH2), YWORD(%%REG_IN), YWORD(%%HK1), 0x11 ; THH = MH*HH %else ; blocks_left == 3 - vmovdqu64 YWORD(%%HK), [%%KP + hashk] - vinserti64x2 %%HK, [%%KP + hashk + 32], 2 - vpclmulqdq %%T0M1, %%REG_IN, %%HK, 0x01 ; M1 = a1*b0 - vpclmulqdq %%T0M2, %%REG_IN, %%HK, 0x10 ; M2 = a0*b1 - vpclmulqdq %%T0H, %%REG_IN, %%HK, 0x11 ; H = a1*b1 - vpclmulqdq %%T0L, %%REG_IN, %%HK, 0x00 ; L = a0*b0 + vmovdqu64 YWORD(%%HK1), [%%KP + hashk] + vmovdqu64 YWORD(%%HK2), [%%KP + hashk + HKeyGap] + vinserti64x2 %%HK1, [%%KP + hashk + 32], 2 + vinserti64x2 %%HK2, [%%KP + hashk + HKeyGap + 32], 2 + vpclmulqdq %%TLL2, %%REG_IN, %%HK2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%REG_IN, %%HK2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%REG_IN, %%HK1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%REG_IN, %%HK1, 0x11 ; THH = MH*HH %endif ; blocks_left -%undef %%REG_IN - -%if %0 == 20 - ;; *** GH/GM/GL passed as arguments -%if %%NUM_BLOCKS >= 4 - ;; add ghash product sums from the first 4, 8 or 12 blocks - vpxorq %%T0M1, %%T0M1, %%T1M1 - vpternlogq %%T0M2, %%GM, %%T1M2, 0x96 - vpternlogq %%T0H, %%GH, %%T1H, 0x96 - vpternlogq %%T0L, %%GL, %%T1L, 0x96 -%else - vpxorq %%T0M1, %%T0M1, %%GM - vpxorq %%T0H, %%T0H, %%GH - vpxorq %%T0L, %%T0L, %%GL -%endif ;; %%NUM_BLOCKS >= 4 -%else - ;; *** GH/GM/GL NOT passed as arguments -%if %%NUM_BLOCKS >= 4 - ;; add ghash product sums from the first 4, 8 or 12 blocks - vpxorq %%T0M1, %%T0M1, %%T1M1 - vpxorq %%T0M2, %%T0M2, %%T1M2 - vpxorq %%T0H, %%T0H, %%T1H - vpxorq %%T0L, %%T0L, %%T1L -%endif ;; %%NUM_BLOCKS >= 4 -%endif ;; %0 == 20 - - ;; integrate TM into TH and TL - vpxorq %%T0M1, %%T0M1, %%T0M2 - vpsrldq %%T1M1, %%T0M1, 8 - vpslldq %%T1M2, %%T0M1, 8 - vpxorq %%T0H, %%T0H, %%T1M1 - vpxorq %%T0L, %%T0L, %%T1M2 + ;; add sums into THH1:TLL1 +%if %%NUM_BLOCKS > 4 + vpternlogq %%TLL1, %%TLL2, %%THL2, 0x96 + vpternlogq %%THH1, %%THH2, %%TLH2, 0x96 %else - ;; ===================================================== - ;; number of blocks is 4, 8, 12 or 16 - ;; T1H/L/M1/M2 include product sums not T0H/L/M1/M2 -%if %0 == 20 - ;; *** GH/GM/GL passed as arguments - vpxorq %%T1M1, %%T1M1, %%GM - vpxorq %%T1H, %%T1H, %%GH - vpxorq %%T1L, %%T1L, %%GL -%endif - ;; integrate TM into TH and TL - vpxorq %%T1M1, %%T1M1, %%T1M2 - vpsrldq %%T0M1, %%T1M1, 8 - vpslldq %%T0M2, %%T1M1, 8 - vpxorq %%T0H, %%T1H, %%T0M1 - vpxorq %%T0L, %%T1L, %%T0M2 + vpxorq %%TLL1, %%TLL2, %%THL2 + vpxorq %%THH1, %%THH2, %%TLH2 +%endif + +%undef %%REG_IN %endif ; blocks_left > 0 - ;; add TH and TL 128-bit words horizontally %if %0 == 20 - ;; *** GH/GM/GL passed as arguments - ;; always do horizontal xor on 4 blocks - VHPXORI4x128 %%T0H, %%T1M1 - VHPXORI4x128 %%T0L, %%T1M2 + ;; *** GH/GL passed as arguments + vpxorq %%TLL1, %%TLL1, %%GL + vpxorq %%THH1, %%THH1, %%GH + ;; new reduction + vpclmulqdq ZWORD(%%GHASH), %%TLL1, [rel POLY], 0x10 + vpshufd %%TLH1, %%TLL1, 01001110b + vpternlogq ZWORD(%%GHASH), %%THH1, %%TLH1, 0x96 + VHPXORI4x128 ZWORD(%%GHASH), %%TLH1 %else - ;; *** GH/GM/GL NOT passed as arguments - ;; - figure out size of horizontal xor + ;; new reduction %if %%NUM_BLOCKS == 1 - ;; do nothing + vpclmulqdq XWORD(%%GHASH), XWORD(%%TLL1), [rel POLY], 0x10 + vpshufd XWORD(%%TLH1), XWORD(%%TLL1), 01001110b + vpternlogq XWORD(%%GHASH), XWORD(%%THH1), XWORD(%%TLH1), 0x96 %elif %%NUM_BLOCKS == 2 - VHPXORI2x128 %%T0H, %%T1M1 - VHPXORI2x128 %%T0L, %%T1M2 + vpclmulqdq YWORD(%%GHASH), YWORD(%%TLL1), [rel POLY], 0x10 + vpshufd YWORD(%%TLH1), YWORD(%%TLL1), 01001110b + vpternlogq YWORD(%%GHASH), YWORD(%%THH1), YWORD(%%TLH1), 0x96 + VHPXORI2x128 YWORD(%%GHASH), YWORD(%%TLH1) %else - VHPXORI4x128 %%T0H, %%T1M1 - VHPXORI4x128 %%T0L, %%T1M2 -%endif + vpclmulqdq ZWORD(%%GHASH), %%TLL1, [rel POLY], 0x10 + vpshufd %%TLH1, %%TLL1, 01001110b + vpternlogq ZWORD(%%GHASH), %%THH1, %%TLH1, 0x96 + VHPXORI4x128 ZWORD(%%GHASH), %%TLH1 %endif +%endif ;; GH/GL passed as arguments - ;; reduction - vmovdqa64 XWORD(%%HK), [rel POLY2] - VCLMUL_REDUCE XWORD(%%GHASH), XWORD(%%HK), \ - XWORD(%%T0H), XWORD(%%T0L), XWORD(%%T0M1), XWORD(%%T0M2) %endmacro ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -639,6 +665,66 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GHASH_MUL2 MACRO to implement: Data*HashKey mod (128,127,126,121,0) +;;; Input: A and B (128-bits each, bit-reflected) +;;; Output: C = A*B*x mod poly, (i.e. >>1 ) +;;; To compute GH = GH*HashKey mod poly, give two constants: +;;; HK = HashKey<<1 mod poly as input +;;; KK = SWAP_H_L( HK_L * POLY) + HK +;;; POLY = 0xC2 << 56 +;;; +;;; Realize four multiplications first, to achieve partially reduced product +;;; TLL = GH_L * KK_L +;;; TLH = GH_L * KK_H +;;; THL = GH_H * HK_L +;;; THH = GH_H * HK_H +;;; +;;; Accumulate results into 2 registers, with corresponding weights +;;; T1 = THH + TLH +;;; T2 = THL + TLL +;;; +;;; Begin reduction +;;; ---------- +;;; | T1 | +;;; --------------- +;;; | T2 | +;;; ---------- +;;; +;;; T3 = SWAP_H_L(T2) +;;; T5 = T2_L * POLY +;;; GH = T1 + T5 + T3 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GHASH_MUL2 7 +%define %%GH %1 ;; [in/out] xmm with multiply operand(s) (128-bits) +%define %%HK %2 ;; [in] xmm with hash key value(s) (128-bits) +%define %%KK %3 ;; [in] xmm with hash key K value(s) (128-bits) +%define %%TLL %4 ;; [clobbered] xmm +%define %%TLH %5 ;; [clobbered] xmm +%define %%THL %6 ;; [clobbered] xmm +%define %%THH %7 ;; [clobbered] xmm + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpclmulqdq %%TLL, %%GH, %%KK, 0x00 ; TLL = GH_L * KK_L + vpclmulqdq %%TLH, %%GH, %%KK, 0x10 ; TLH = GH_L * KK_H + vpclmulqdq %%THL, %%GH, %%HK, 0x01 ; THL = GH_H * HK_L + vpclmulqdq %%THH, %%GH, %%HK, 0x11 ; THH = GH_H * HK_H + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; add products + vpxorq %%TLL, %%TLL, %%THL + vpxorq %%THH, %%THH, %%TLH + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; new reduction + vpclmulqdq %%GH, %%TLL, [rel POLY], 0x10 + vpshufd %%TLH, %%TLL, 01001110b + vpternlogq %%GH, %%THH, %%TLH, 0x96 + + ; @note: it can support YMM or ZMM but horizontal XOR would be required here + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%endmacro + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx512 ;;; functions, but are kept to allow users to switch cpu architectures between calls @@ -646,14 +732,14 @@ %macro PRECOMPUTE 10 %define %%GDATA %1 ;; [in/out] GPR, pointer to GCM key data structure, content updated %define %%HK %2 ;; [in] xmm, hash key -%define %%T1 %3 ;; [clobbered] xmm -%define %%T2 %4 ;; [clobbered] xmm +%define %%T1 %3 ;; [clobbered**] xmm +%define %%T2 %4 ;; [clobbered**] xmm %define %%T3 %5 ;; [clobbered] xmm %define %%T4 %6 ;; [clobbered] xmm -%define %%T5 %7 ;; [clobbered] xmm +%define %%T5 %7 ;; [clobbered**] xmm %define %%T6 %8 ;; [clobbered] xmm -%define %%T7 %9 ;; [clobbered] xmm -%define %%T8 %10 ;; [clobbered] xmm +%define %%T7 %9 ;; [clobbered**] xmm +%define %%T8 %10 ;; [clobbered**] xmm %xdefine %%ZT1 ZWORD(%%T1) %xdefine %%ZT2 ZWORD(%%T2) @@ -681,28 +767,58 @@ GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 vmovdqu64 [%%GDATA + HashKey_4], %%T5 vinserti64x2 %%ZT7, %%T5, 0 + ;; **ZT5 amd ZT7 to be cleared (hash key) + + ;; calculate HashKeyK = HashKey x POLY + vpclmulqdq %%ZT1, %%ZT7, [rel POLY], 0x10 + vpshufd %%ZT2, %%ZT7, 01001110b + vpxorq %%ZT1, %%ZT1, %%ZT2 + vmovdqu64 [%%GDATA + HashKeyK_4], %%ZT1 + ;; **ZT1 amd ZT2 to be cleared (hash key) ;; switch to 4x128-bit computations now vshufi64x2 %%ZT5, %%ZT5, %%ZT5, 0x00 ;; broadcast HashKey^4 across all ZT5 vmovdqa64 %%ZT8, %%ZT7 ;; save HashKey^4 to HashKey^1 in ZT8 + ;; **ZT8 to be cleared (hash key) ;; calculate HashKey^5<<1 mod poly, HashKey^6<<1 mod poly, ... HashKey^8<<1 mod poly GHASH_MUL %%ZT7, %%ZT5, %%ZT1, %%ZT3, %%ZT4, %%ZT6, %%ZT2 vmovdqu64 [%%GDATA + HashKey_8], %%ZT7 ;; HashKey^8 to HashKey^5 in ZT7 now + + ;; calculate HashKeyX = HashKey x POLY + vpclmulqdq %%ZT1, %%ZT7, [rel POLY], 0x10 + vpshufd %%ZT2, %%ZT7, 01001110b + vpxorq %%ZT1, %%ZT1, %%ZT2 + vmovdqu64 [%%GDATA + HashKeyK_8], %%ZT1 + vshufi64x2 %%ZT5, %%ZT7, %%ZT7, 0x00 ;; broadcast HashKey^8 across all ZT5 ;; calculate HashKey^9<<1 mod poly, HashKey^10<<1 mod poly, ... HashKey^48<<1 mod poly ;; use HashKey^8 as multiplier against ZT8 and ZT7 - this allows deeper ooo execution %assign i 12 -%rep ((48 - 8) / 8) +%rep ((big_loop_nblocks - 8) / 8) ;; compute HashKey^(4 + n), HashKey^(3 + n), ... HashKey^(1 + n) GHASH_MUL %%ZT8, %%ZT5, %%ZT1, %%ZT3, %%ZT4, %%ZT6, %%ZT2 vmovdqu64 [%%GDATA + HashKey_ %+ i], %%ZT8 + + ;; calculate HashKeyK = HashKey x POLY + vpclmulqdq %%ZT1, %%ZT8, [rel POLY], 0x10 + vpshufd %%ZT2, %%ZT8, 01001110b + vpxorq %%ZT1, %%ZT1, %%ZT2 + vmovdqu64 [%%GDATA + HashKeyK_ %+ i], %%ZT1 + %assign i (i + 4) ;; compute HashKey^(8 + n), HashKey^(7 + n), ... HashKey^(5 + n) GHASH_MUL %%ZT7, %%ZT5, %%ZT1, %%ZT3, %%ZT4, %%ZT6, %%ZT2 vmovdqu64 [%%GDATA + HashKey_ %+ i], %%ZT7 + + ;; calculate HashKeyK = HashKey x POLY + vpclmulqdq %%ZT1, %%ZT7, [rel POLY], 0x10 + vpshufd %%ZT2, %%ZT7, 01001110b + vpxorq %%ZT1, %%ZT1, %%ZT2 + vmovdqu64 [%%GDATA + HashKeyK_ %+ i], %%ZT1 + %assign i (i + 4) %endrep %endmacro @@ -718,10 +834,10 @@ %define %%AAD_HASH %3 ; [in/out] xmm ghash value %define %%GDATA_KEY %4 ; [in] pointer to keys %define %%ZT0 %5 ; [clobbered] ZMM register -%define %%ZT1 %6 ; [clobbered] ZMM register -%define %%ZT2 %7 ; [clobbered] ZMM register -%define %%ZT3 %8 ; [clobbered] ZMM register -%define %%ZT4 %9 ; [clobbered] ZMM register +%define %%ZT1 %6 ; [clobbered**] ZMM register +%define %%ZT2 %7 ; [clobbered**] ZMM register +%define %%ZT3 %8 ; [clobbered**] ZMM register +%define %%ZT4 %9 ; [clobbered**] ZMM register %define %%ZT5 %10 ; [clobbered] ZMM register %define %%ZT6 %11 ; [clobbered] ZMM register %define %%ZT7 %12 ; [clobbered] ZMM register @@ -750,79 +866,53 @@ vmovdqa64 %%SHFMSK, [rel SHUF_MASK] -%%_get_AAD_loop48x16: - cmp %%T2, (48*16) - jb %%_exit_AAD_loop48x16 +%%_get_AAD_loop2x32x16: + cmp %%T2, (2*32*16) + jb %%_get_AAD_loop32x16 - GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ - %%T1, (0*64), 0, \ - %%GDATA_KEY, HashKey_48, 0, ZWORD(%%AAD_HASH), \ - %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK - - GHASH_16 mid, %%ZT5, %%ZT6, %%ZT7, \ - %%T1, (4*64), 0, \ - %%GDATA_KEY, HashKey_32, 0, NO_HASH_IN_OUT, \ + GHASH_16 start, hk_bcast, %%ZT5, %%ZT6, \ + %%T1, (0*16*16), 0, \ + %%GDATA_KEY, HashKey_32, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%ZT7, %%SHFMSK - GHASH_16 end_reduce, %%ZT5, %%ZT6, %%ZT7, \ - %%T1, (8*64), 0, \ + GHASH_16 end_reduce_no_hxor, hk_bcast, %%ZT5, %%ZT6, \ + %%T1, (1*16*16), 0, \ %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK - - sub %%T2, (12*64) - je %%_CALC_AAD_done - - add %%T1, (12*64) - jmp %%_get_AAD_loop48x16 + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%ZT7, %%SHFMSK + ;; **ZT1 can potentially include clear text, ZT16 & ZT17 hash key + add %%T1, (32*16) + sub %%T2, (32*16) + jmp %%_get_AAD_loop2x32x16 -%%_exit_AAD_loop48x16: - ; Less than 48x16 bytes remaining - cmp %%T2, (16*16) - jb %%_less_than_16x16 - je %%_equal_16x16 +%%_get_AAD_loop32x16: cmp %%T2, (32*16) - jb %%_less_than_32x16 - je %%_equal_32x16 - - ;; calculate offset to hash key to start with - lea %%T3, [%%T2 + 15] - and %%T3, ~15 - neg %%T3 - add %%T3, HashKey_1 + 16 + jb %%_exit_AAD_loop32x16 - ; Get next 16 blocks - GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ - %%T1, (0*64), 0, \ - %%GDATA_KEY, %%T3, 0, ZWORD(%%AAD_HASH), \ + GHASH_16 start, hk_load, %%ZT5, %%ZT6, \ + %%T1, (0*16*16), 0, \ + %%GDATA_KEY, HashKey_32, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%ZT7, %%SHFMSK - GHASH_16 mid, %%ZT5, %%ZT6, %%ZT7, \ - %%T1, (4*64), 0, \ - %%GDATA_KEY, {%%T3 + 16*16}, 0, ZWORD(%%AAD_HASH), \ + GHASH_16 end_reduce, hk_load, %%ZT5, %%ZT6, \ + %%T1, (1*16*16), 0, \ + %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK - + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%ZT7, %%SHFMSK + ;; **ZT1 can potentially include clear text, ZT16 & ZT17 hash key sub %%T2, (32*16) - add %%T1, (32*16) - jmp %%_less_than_16x16_remain + je %%_CALC_AAD_done -%%_equal_32x16: - GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ - %%T1, (0*64), 0, \ - %%GDATA_KEY, HashKey_32, 0, ZWORD(%%AAD_HASH), \ - %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + add %%T1, (32*16) + jmp %%_get_AAD_loop32x16 - GHASH_16 end_reduce, %%ZT5, %%ZT6, %%ZT7, \ - %%T1, (4*64), 0, \ - %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ - %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK - jmp %%_CALC_AAD_done +%%_exit_AAD_loop32x16: + ; Less than 32x16 bytes remaining + cmp %%T2, (16*16) + jb %%_less_than_16x16 + je %%_equal_16x16 %%_less_than_32x16: ;; calculate offset to hash key to start with @@ -831,26 +921,28 @@ neg %%T3 add %%T3, HashKey_1 + 16 - GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ + GHASH_16 start, hk_load, %%ZT5, %%ZT6, \ %%T1, (0*64), 0, \ %%GDATA_KEY, %%T3, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%ZT7, %%SHFMSK + ;; **ZT1 can potentially include clear text, ZT16 & ZT17 hash key sub %%T2, (16*16) add %%T1, (16*16) jmp %%_less_than_16x16_remain %%_equal_16x16: - GHASH_16 start_reduce, %%ZT5, %%ZT6, %%ZT7, \ + GHASH_16 start_reduce, hk_load, %%ZT5, %%ZT6, \ %%T1, (0*64), 0, \ %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%ZT7, %%SHFMSK + ;; **ZT1 can potentially include clear text, ZT16 & ZT17 hash key jmp %%_CALC_AAD_done ; Less than 16x16 bytes remaining %%_less_than_16x16_remain: - ;; ZT5 (H), ZT6 (M), ZT7 (L) contain ghash sums + ;; ZT5 (H), ZT6 (L) contain ghash sums ;; prep mask source address lea %%T3, [rel byte64_len_to_mask_table] lea %%T3, [%%T3 + %%T2*8] @@ -907,19 +999,21 @@ ZMM_LOAD_MASKED_BLOCKS_0_16 \ I, %%T1, 0, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT17, %%MASKREG + %%ZT1, %%ZT2, %%ZT3, %%ZT4, %%MASKREG ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 \ I, vpshufb, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ - %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ %%SHFMSK, %%SHFMSK, %%SHFMSK, %%SHFMSK + ;; **ZT1, ZT2, ZT3, ZT4 contain clear text GHASH_1_TO_16 %%GDATA_KEY, ZWORD(%%AAD_HASH), \ - %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ - %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - ZWORD(%%AAD_HASH), %%ZT14, %%ZT15, %%ZT16, %%ZT17, I, \ - %%ZT5, %%ZT6, %%ZT7 + %%ZT0, %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ + %%ZT9, %%ZT10, %%ZT11, %%ZT12, %%ZT7, \ + ZWORD(%%AAD_HASH), %%ZT1, %%ZT2, %%ZT3, %%ZT4, I, \ + %%ZT5, %%ZT6 + ;; **ZT16 and ZT17 may not get cleared above when number of blocks is below 4 jmp %%_CALC_AAD_done %assign I (I - 1) @@ -998,10 +1092,11 @@ %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ %%SHFMSK, %%SHFMSK, %%SHFMSK, %%SHFMSK + ;; **ZT1, ZT2, ZT3, ZT4 contain clear text GHASH_1_TO_16 %%GDATA_KEY, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT5, %%ZT6, %%ZT7, %%ZT8, \ - %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT9, %%ZT10, %%ZT11, %%ZT12, %%ZT13, \ ZWORD(%%AAD_HASH), %%ZT1, %%ZT2, %%ZT3, %%ZT4, I %if I > 1 ;; fall through to CALC_AAD_done in 1 block case @@ -1038,7 +1133,7 @@ %define %%GPTMP1 %10 ; [clobbered] GP temporary register %define %%GPTMP2 %11 ; [clobbered] GP temporary register %define %%ZTMP0 %12 ; [clobbered] ZMM temporary register -%define %%ZTMP1 %13 ; [clobbered] ZMM temporary register +%define %%ZTMP1 %13 ; [clobbered**] ZMM temporary register %define %%ZTMP2 %14 ; [clobbered] ZMM temporary register %define %%ZTMP3 %15 ; [clobbered] ZMM temporary register %define %%ZTMP4 %16 ; [clobbered] ZMM temporary register @@ -1072,10 +1167,12 @@ READ_SMALL_DATA_INPUT_LEN_BT16_AVX512 %%XTMP0, %%PLAIN_CIPH_IN, %%PLAIN_CIPH_LEN, \ %%IA0, %%IA2, %%MASKREG + ;; **XTMP0 includes plain text ;; XTMP1 = my_ctx_data.partial_block_enc_key vmovdqu64 %%XTMP1, [%%GDATA_CTX + PBlockEncKey] - vmovdqu64 %%XTMP2, [%%GDATA_KEY + HashKey] + vmovdqu64 %%XTMP2, [%%GDATA_KEY + HashKey_1] + vmovdqu64 %%XTMP9, [%%GDATA_KEY + HashKey_1 + HKeyGap] ;; adjust the shuffle mask pointer to be able to shift right %%LENGTH bytes ;; (16 - %%LENGTH) is the number of bytes in plaintext mod 16) @@ -1087,8 +1184,10 @@ %ifidn %%ENC_DEC, DEC ;; keep copy of cipher text in %%XTMP4 vmovdqa64 %%XTMP4, %%XTMP0 + ;; **XTMP4 includes cipher text (not sensitive) %endif vpxorq %%XTMP1, %%XTMP0 ; Ciphertext XOR E(K, Yn) + ;; **XTMP1 may contain clear text (decrypt direction) ;; Set %%IA1 to be the amount of data left in CIPH_PLAIN_IN after filling the block ;; Determine if partial block is not being filled and shift mask accordingly @@ -1105,6 +1204,7 @@ ;; get the appropriate mask to mask out bottom %%LENGTH bytes of %%XTMP1 ;; - mask out bottom %%LENGTH bytes of %%XTMP1 vmovdqu64 %%XTMP0, [%%IA0 + ALL_F - SHIFT_MASK] + ;; **XTMP0 (potentially clear text) gets cleared with a shift mask vpand %%XTMP1, %%XTMP0 %ifidn %%ENC_DEC, DEC @@ -1121,7 +1221,7 @@ jl %%_partial_incomplete ;; GHASH computation for the last <16 Byte block - GHASH_MUL %%AAD_HASH, %%XTMP2, %%XTMP5, %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9 + GHASH_MUL2 %%AAD_HASH, %%XTMP2, %%XTMP9, %%XTMP5, %%XTMP6, %%XTMP7, %%XTMP8 mov qword [%%GDATA_CTX + PBlockLen], 0 @@ -1177,11 +1277,11 @@ %define %%DAT3 %14 ; [out] ZMM with cipher text shuffled for GHASH %define %%LAST_CIPHER_BLK %15 ; [out] XMM to put ciphered counter block partially xor'ed with text %define %%LAST_GHASH_BLK %16 ; [out] XMM to put last cipher text block shuffled for GHASH -%define %%CTR0 %17 ; [clobbered] ZMM temporary -%define %%CTR1 %18 ; [clobbered] ZMM temporary -%define %%CTR2 %19 ; [clobbered] ZMM temporary -%define %%CTR3 %20 ; [clobbered] ZMM temporary -%define %%ZT1 %21 ; [clobbered] ZMM temporary +%define %%CTR0 %17 ; [clobbered**] ZMM temporary +%define %%CTR1 %18 ; [clobbered**] ZMM temporary +%define %%CTR2 %19 ; [clobbered**] ZMM temporary +%define %%CTR3 %20 ; [clobbered**] ZMM temporary +%define %%ZT1 %21 ; [clobbered**] ZMM temporary %define %%IA0 %22 ; [clobbered] GP temporary %define %%IA1 %23 ; [clobbered] GP temporary %define %%MASKREG %24 ; [clobbered] mask register @@ -1258,6 +1358,7 @@ %%NUM_BLOCKS, NROUNDS %assign j (j + 1) %endrep + ;; **DAT0, DAT1, DAT2, DAT3 may contain clear text %ifidn %%INSTANCE_TYPE, multi_call ;; retrieve the last cipher counter block (partially XOR'ed with text) @@ -1275,6 +1376,7 @@ ;; write cipher/plain text back to output and ZMM_STORE_MASKED_BLOCKS_0_16 %%NUM_BLOCKS, %%CIPH_PLAIN_OUT, %%DATA_OFFSET, \ %%CTR0, %%CTR1, %%CTR2, %%CTR3, %%MASKREG + ;; **CTR0, CTR1, CTR2, CTR3 may contain clear text ;; zero bytes outside the mask before hashing %if %%NUM_BLOCKS <= 4 @@ -1304,6 +1406,7 @@ %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK %endif ; Encrypt + ;; **DAT0, DAT1, DAT2, DAT3 overwritten with shuffled cipher text %ifidn %%INSTANCE_TYPE, multi_call ;; Extract the last block for partials and multi_call cases @@ -1322,7 +1425,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Computes GHASH on 1 to 16 blocks -%macro INITIAL_BLOCKS_PARTIAL_GHASH 22-25 +%macro INITIAL_BLOCKS_PARTIAL_GHASH 23-25 %define %%GDATA_KEY %1 ; [in] key pointer %define %%GDATA_CTX %2 ; [in] context pointer %define %%LENGTH %3 ; [in/clobbered] length in bytes @@ -1345,8 +1448,8 @@ %define %%ZT6 %20 ; [clobbered] ZMM temporary %define %%ZT7 %21 ; [clobbered] ZMM temporary %define %%ZT8 %22 ; [clobbered] ZMM temporary -%define %%GH %23 ; [in] ZMM with hi product part -%define %%GM %24 ; [in] ZMM with mid prodcut part +%define %%ZT9 %23 ; [clobbered] ZMM temporary +%define %%GH %24 ; [in] ZMM with hi product part %define %%GL %25 ; [in] ZMM with lo product part %ifidn %%INSTANCE_TYPE, single_call @@ -1355,19 +1458,20 @@ ;;; - hash all data including partial block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%if %0 == 22 +%if %0 == 23 ;; start GHASH compute GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ - %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, ZWORD(%%HASH_IN_OUT), \ %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS %elif %0 == 25 ;; continue GHASH compute GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ - %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ - %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS, %%GH, %%GM, %%GL + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS, %%GH, %%GL %endif + ;; **DAT0, DAT1, DAT2, DAT3 are OK here - they contain shuffled cipher text %else ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1395,22 +1499,24 @@ mov qword [%%GDATA_CTX + PBlockLen], 0 ;; Hash all of the data -%if %0 == 22 +%if %0 == 23 ;; start GHASH compute GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ - %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, ZWORD(%%HASH_IN_OUT), \ %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS %elif %0 == 25 ;; continue GHASH compute GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ - %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ - %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS, %%GH, %%GM, %%GL + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS, %%GH, %%GL %endif jmp %%_small_initial_compute_done %endif ; %if %%NUM_BLOCKS < 16 + ;; **DAT0, DAT1, DAT2, DAT3 are OK here - they contain shuffled cipher text + %%_small_initial_partial_block: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1431,38 +1537,29 @@ %if (%%NUM_BLOCKS > last_block_to_hash) ;; ZT12-ZT20 - temporary registers -%if %0 == 22 +%if %0 == 23 ;; start GHASH compute GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ - %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, ZWORD(%%HASH_IN_OUT), \ %%DAT0, %%DAT1, %%DAT2, %%DAT3, k %elif %0 == 25 ;; continue GHASH compute GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ - %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ - %%DAT0, %%DAT1, %%DAT2, %%DAT3, k, %%GH, %%GM, %%GL + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, k, %%GH, %%GL %endif + ;; **DAT0, DAT1, DAT2, DAT3 are OK here - they contain shuffled cipher text ;; just fall through no jmp needed %else %if %0 == 25 ;; Reduction is required in this case. - ;; Integrate GM into GH and GL. - vpsrldq %%ZT0, %%GM, 8 - vpslldq %%ZT1, %%GM, 8 - vpxorq %%GH, %%GH, %%ZT0 - vpxorq %%GL, %%GL, %%ZT1 - - ;; Add GH and GL 128-bit words horizontally - VHPXORI4x128 %%GH, %%ZT0 - VHPXORI4x128 %%GL, %%ZT1 - - ;; 256-bit to 128-bit reduction - vmovdqa64 XWORD(%%ZT0), [rel POLY2] - VCLMUL_REDUCE XWORD(%%HASH_IN_OUT), XWORD(%%ZT0), \ - XWORD(%%GH), XWORD(%%GL), XWORD(%%ZT1), XWORD(%%ZT2) + vpclmulqdq ZWORD(%%HASH_IN_OUT), %%GL, [rel POLY], 0x10 + vpshufd %%ZT0, %%GL, 01001110b + vpternlogq ZWORD(%%HASH_IN_OUT), %%ZT0, %%GH, 0x96 + VHPXORI4x128 ZWORD(%%HASH_IN_OUT), %%ZT0 %endif ;; Record that a reduction is not needed - ;; In this case no hashes are computed because there @@ -1516,7 +1613,7 @@ ;;; ;;; num_initial_blocks is expected to include the partial final block ;;; in the count. -%macro INITIAL_BLOCKS_PARTIAL 30 +%macro INITIAL_BLOCKS_PARTIAL 31 %define %%GDATA_KEY %1 ; [in] key pointer %define %%GDATA_CTX %2 ; [in] context pointer %define %%CIPH_PLAIN_OUT %3 ; [in] text output pointer @@ -1529,24 +1626,25 @@ %define %%ENC_DEC %10 ; [in] cipher direction (ENC/DEC) %define %%INSTANCE_TYPE %11 ; [in] multi_call or single_call %define %%CTR0 %12 ; [clobbered] ZMM temporary -%define %%CTR1 %13 ; [clobbered] ZMM temporary -%define %%CTR2 %14 ; [clobbered] ZMM temporary -%define %%CTR3 %15 ; [clobbered] ZMM temporary +%define %%CTR1 %13 ; [clobbered**] ZMM temporary +%define %%CTR2 %14 ; [clobbered**] ZMM temporary +%define %%CTR3 %15 ; [clobbered**] ZMM temporary %define %%DAT0 %16 ; [clobbered] ZMM temporary %define %%DAT1 %17 ; [clobbered] ZMM temporary %define %%DAT2 %18 ; [clobbered] ZMM temporary %define %%DAT3 %19 ; [clobbered] ZMM temporary %define %%LAST_CIPHER_BLK %20 ; [clobbered] ZMM temporary %define %%LAST_GHASH_BLK %21 ; [clobbered] ZMM temporary -%define %%ZT0 %22 ; [clobbered] ZMM temporary +%define %%ZT0 %22 ; [clobbered**] ZMM temporary %define %%ZT1 %23 ; [clobbered] ZMM temporary %define %%ZT2 %24 ; [clobbered] ZMM temporary %define %%ZT3 %25 ; [clobbered] ZMM temporary %define %%ZT4 %26 ; [clobbered] ZMM temporary -%define %%IA0 %27 ; [clobbered] GP temporary -%define %%IA1 %28 ; [clobbered] GP temporary -%define %%MASKREG %29 ; [clobbered] mask register -%define %%SHUFMASK %30 ; [clobbered] ZMM for BE/LE shuffle mask +%define %%ZT5 %27 ; [clobbered] ZMM temporary +%define %%IA0 %28 ; [clobbered] GP temporary +%define %%IA1 %29 ; [clobbered] GP temporary +%define %%MASKREG %30 ; [clobbered] mask register +%define %%SHUFMASK %31 ; [clobbered] ZMM for BE/LE shuffle mask INITIAL_BLOCKS_PARTIAL_CIPHER \ %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ @@ -1555,14 +1653,16 @@ XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ %%CTR0, %%CTR1, %%CTR2, %%CTR3, %%ZT0, \ %%IA0, %%IA1, %%MASKREG, %%SHUFMASK - + ;; **CTR0, CTR1, CTR2, CTR3, %%ZT0 may contain sensitive data INITIAL_BLOCKS_PARTIAL_GHASH \ %%GDATA_KEY, %%GDATA_CTX, %%LENGTH, \ %%NUM_BLOCKS, %%HASH_IN_OUT, %%ENC_DEC, \ %%INSTANCE_TYPE, %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ %%CTR0, %%CTR1, %%CTR2, %%CTR3, %%ZT0, %%ZT1, \ - %%ZT2, %%ZT3, %%ZT4 + %%ZT2, %%ZT3, %%ZT4, %%ZT5 + ;; **CTR1 may not get cleared (message below 4 blocks) and it may still contain sensitive data + ;; **CTR1, CTR2, CTR3 may not get cleared (message below 16 bytes & SGL) and may still contain sensitive data %endmacro ; INITIAL_BLOCKS_PARTIAL @@ -1583,34 +1683,34 @@ %define %%GHASHIN_BLK_OFFSET %10 ; [in] numerical offset for GHASH blocks in %define %%SHFMSK %11 ; [in] ZMM with byte swap mask for pshufb %define %%B00_03 %12 ; [clobbered] temporary ZMM -%define %%B04_07 %13 ; [clobbered] temporary ZMM +%define %%B04_07 %13 ; [clobbered**] temporary ZMM %define %%B08_11 %14 ; [clobbered] temporary ZMM %define %%B12_15 %15 ; [clobbered] temporary ZMM -%define %%GH1H_UNUSED %16 ; [clobbered] temporary ZMM -%define %%GH1L %17 ; [clobbered] temporary ZMM -%define %%GH1M %18 ; [clobbered] temporary ZMM -%define %%GH1T %19 ; [clobbered] temporary ZMM -%define %%GH2H %20 ; [clobbered] temporary ZMM -%define %%GH2L %21 ; [clobbered] temporary ZMM -%define %%GH2M %22 ; [clobbered] temporary ZMM -%define %%GH2T %23 ; [clobbered] temporary ZMM -%define %%GH3H %24 ; [clobbered] temporary ZMM -%define %%GH3L %25 ; [clobbered] temporary ZMM -%define %%GH3M %26 ; [clobbered] temporary ZMM -%define %%GH3T %27 ; [clobbered] temporary ZMM -%define %%AESKEY1 %28 ; [clobbered] temporary ZMM -%define %%AESKEY2 %29 ; [clobbered] temporary ZMM +%define %%GHKEY3 %16 ; [clobbered] temporary ZMM +%define %%TLL1 %17 ; [clobbered] temporary ZMM +%define %%TLL2 %18 ; [clobbered] temporary ZMM +%define %%TLL3 %19 ; [clobbered] temporary ZMM +%define %%TLH1 %20 ; [clobbered] temporary ZMM +%define %%TLH2 %21 ; [clobbered] temporary ZMM +%define %%TLH3 %22 ; [clobbered] temporary ZMM +%define %%THL1 %23 ; [clobbered] temporary ZMM +%define %%THL2 %24 ; [clobbered] temporary ZMM +%define %%THL3 %25 ; [clobbered] temporary ZMM +%define %%THH2 %26 ; [clobbered] temporary ZMM +%define %%THH3 %27 ; [clobbered] temporary ZMM +%define %%AESKEY1 %28 ; [clobbered**] temporary ZMM +%define %%AESKEY2 %29 ; [clobbered**] temporary ZMM %define %%GHKEY1 %30 ; [clobbered] temporary ZMM %define %%GHKEY2 %31 ; [clobbered] temporary ZMM %define %%GHDAT1 %32 ; [clobbered] temporary ZMM %define %%GHDAT2 %33 ; [clobbered] temporary ZMM %define %%ZT01 %34 ; [clobbered] temporary ZMM -%define %%ADDBE_4x4 %35 ; [in] ZMM with 4x128bits 4 in big-endian -%define %%ADDBE_1234 %36 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian -%define %%GHASH_TYPE %37 ; [in] "start", "start_reduce", "mid", "end_reduce" -%define %%TO_REDUCE_L %38 ; [in] ZMM for low 4x128-bit GHASH sum -%define %%TO_REDUCE_H %39 ; [in] ZMM for hi 4x128-bit GHASH sum -%define %%TO_REDUCE_M %40 ; [in] ZMM for medium 4x128-bit GHASH sum +%define %%GHKEY4 %35 ; [clobbered] temporary ZMM +%define %%ADDBE_4x4 %36 ; [in] ZMM with 4x128bits 4 in big-endian +%define %%ADDBE_1234 %37 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian +%define %%GHASH_TYPE %38 ; [in] "start", "start_reduce", "mid", "end_reduce" +%define %%TO_REDUCE_L %39 ; [in] ZMM for low 4x128-bit GHASH sum +%define %%TO_REDUCE_H %40 ; [in] ZMM for hi 4x128-bit GHASH sum %define %%ENC_DEC %41 ; [in] cipher direction %define %%HASH_IN_OUT %42 ; [in/out] XMM ghash in/out value %define %%IA0 %43 ; [clobbered] GP temporary @@ -1619,20 +1719,18 @@ %define %%NUM_BLOCKS %46 ; [in] numerical value with number of blocks to be encrypted/ghashed (1 to 16) %define %%INSTANCE_TYPE %47 ; [in] multi_call or single_call -%xdefine %%GH1H %%HASH_IN_OUT ; this is to avoid additional move in do_reduction case +%xdefine %%THH1 %%HASH_IN_OUT ; this is to avoid additional move in do_reduction case -%define %%LAST_GHASH_BLK %%GH1L -%define %%LAST_CIPHER_BLK %%GH1T +%define %%LAST_GHASH_BLK %%THL1 +%define %%LAST_CIPHER_BLK %%TLH1 -%define %%RED_POLY %%GH2T -%define %%RED_P1 %%GH2L -%define %%RED_T1 %%GH2H -%define %%RED_T2 %%GH2M +%define %%RED_T1 %%THH2 +%define %%RED_T2 %%TLL2 -%define %%DATA1 %%GH3H -%define %%DATA2 %%GH3L -%define %%DATA3 %%GH3M -%define %%DATA4 %%GH3T +%define %%DATA1 %%THH3 +%define %%DATA2 %%THL3 +%define %%DATA3 %%TLH3 +%define %%DATA4 %%TLL3 ;; do reduction after the 16 blocks ? %assign do_reduction 0 @@ -1709,6 +1807,7 @@ vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] %endif vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (0*64)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (0*64) + HKeyGap] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; save counter for the next round @@ -1729,7 +1828,8 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; pre-load constants vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 1)] - vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] + vmovdqu64 %%GHKEY3, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] + vmovdqu64 %%GHKEY4, [%%GDATA + %%HASHKEY_OFFSET + (1*64) + HKeyGap] vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (1*64)] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1746,12 +1846,13 @@ ;;================================================== ;; GHASH 4 blocks (15 to 12) - vpclmulqdq %%GH1H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 - vpclmulqdq %%GH1L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 - vpclmulqdq %%GH1M, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 - vpclmulqdq %%GH1T, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 + vpclmulqdq %%TLL1, %%GHDAT1, %%GHKEY2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH1, %%GHDAT1, %%GHKEY2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL1, %%GHDAT1, %%GHKEY1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH1, %%GHDAT1, %%GHKEY1, 0x11 ; THH = MH*HH vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (2*64)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (2*64) + HKeyGap] vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (2*64)] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1764,12 +1865,13 @@ ;; ================================================= ;; GHASH 4 blocks (11 to 8) - vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 - vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 - vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 - vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + vpclmulqdq %%TLL2, %%GHDAT2, %%GHKEY4, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%GHDAT2, %%GHKEY4, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%GHDAT2, %%GHKEY3, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%GHDAT2, %%GHKEY3, 0x11 ; THH = MH*HH - vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vmovdqu64 %%GHKEY3, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vmovdqu64 %%GHKEY4, [%%GDATA + %%HASHKEY_OFFSET + (3*64) + HKeyGap] vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (3*64)] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1782,10 +1884,10 @@ ;; ================================================= ;; GHASH 4 blocks (7 to 4) - vpclmulqdq %%GH3M, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 - vpclmulqdq %%GH3T, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 - vpclmulqdq %%GH3H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 - vpclmulqdq %%GH3L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + vpclmulqdq %%TLL3, %%GHDAT1, %%GHKEY2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH3, %%GHDAT1, %%GHKEY2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL3, %%GHDAT1, %%GHKEY1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH3, %%GHDAT1, %%GHKEY1, 0x11 ; THH = MH*HH ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES rounds 3 @@ -1797,10 +1899,10 @@ ;; ================================================= ;; Gather (XOR) GHASH for 12 blocks - vpternlogq %%GH1H, %%GH2H, %%GH3H, 0x96 - vpternlogq %%GH1L, %%GH2L, %%GH3L, 0x96 - vpternlogq %%GH1T, %%GH2T, %%GH3T, 0x96 - vpternlogq %%GH1M, %%GH2M, %%GH3M, 0x96 + vpternlogq %%TLL1, %%TLL2, %%TLL3, 0x96 + vpternlogq %%TLH1, %%TLH2, %%TLH3, 0x96 + vpternlogq %%THL1, %%THL2, %%THL3, 0x96 + vpternlogq %%THH1, %%THH2, %%THH3, 0x96 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES rounds 4 @@ -1825,10 +1927,10 @@ ;; ================================================= ;; GHASH 4 blocks (3 to 0) - vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 - vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 - vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 - vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + vpclmulqdq %%TLL2, %%GHDAT2, %%GHKEY4, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%GHDAT2, %%GHKEY4, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%GHDAT2, %%GHKEY3, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%GHDAT2, %%GHKEY3, 0x11 ; THH = MH*HH ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES round 6 @@ -1839,31 +1941,18 @@ vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 8)] ;; ================================================= - ;; gather GHASH in GH1L (low), GH1H (high), GH1M (mid) - ;; - add GH2[MTLH] to GH1[MTLH] - vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 -%if do_reduction != 0 - -%if is_start != 0 - vpxorq %%GH1M, %%GH1M, %%GH2M -%else - vpternlogq %%GH1H, %%TO_REDUCE_H, %%GH2H, 0x96 - vpternlogq %%GH1L, %%TO_REDUCE_L, %%GH2L, 0x96 - vpternlogq %%GH1M, %%TO_REDUCE_M, %%GH2M, 0x96 -%endif - -%else - ;; Update H/M/L hash sums if not carrying reduction + ;; gather GHASH sums into TO_REDUCE_[HL] %if is_start != 0 - vpxorq %%TO_REDUCE_H, %%GH1H, %%GH2H - vpxorq %%TO_REDUCE_L, %%GH1L, %%GH2L - vpxorq %%TO_REDUCE_M, %%GH1M, %%GH2M + vpxorq %%TO_REDUCE_L, %%TLL2, %%THL2 + vpxorq %%TO_REDUCE_H, %%THH2, %%TLH2 + vpternlogq %%TO_REDUCE_L, %%TLL1, %%THL1, 0x96 + vpternlogq %%TO_REDUCE_H, %%THH1, %%TLH1, 0x96 %else - vpternlogq %%TO_REDUCE_H, %%GH1H, %%GH2H, 0x96 - vpternlogq %%TO_REDUCE_L, %%GH1L, %%GH2L, 0x96 - vpternlogq %%TO_REDUCE_M, %%GH1M, %%GH2M, 0x96 -%endif - + ;; not the first round so sums need to be updated + vpternlogq %%TO_REDUCE_L, %%TLL2, %%THL2, 0x96 + vpternlogq %%TO_REDUCE_H, %%THH2, %%TLH2, 0x96 + vpternlogq %%TO_REDUCE_L, %%TLL1, %%THL1, 0x96 + vpternlogq %%TO_REDUCE_H, %%THH1, %%TLH1, 0x96 %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1875,13 +1964,11 @@ vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 9)] ;; ================================================= - ;; prepare mid sum for adding to high & low - ;; load polynomial constant for reduction + ;; new reduction (result in THH1) %if do_reduction != 0 - vpsrldq %%GH2M, %%GH1M, 8 - vpslldq %%GH1M, %%GH1M, 8 - - vmovdqa64 XWORD(%%RED_POLY), [rel POLY2] + vpclmulqdq %%THH1, %%TO_REDUCE_L, [rel POLY], 0x10 + vpshufd %%THL1, %%TO_REDUCE_L, 01001110b + vpternlogq %%THH1, %%THL1, %%TO_REDUCE_H, 0x96 %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES round 8 @@ -1892,15 +1979,9 @@ vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 10)] ;; ================================================= - ;; Add mid product to high and low + ;; horizontal xor of 4 reduced hashes %if do_reduction != 0 -%if is_start != 0 - vpternlogq %%GH1H, %%GH2H, %%GH2M, 0x96 ; TH = TH1 + TH2 + TM>>64 - vpternlogq %%GH1L, %%GH2L, %%GH1M, 0x96 ; TL = TL1 + TL2 + TM<<64 -%else - vpxorq %%GH1H, %%GH1H, %%GH2M ; TH = TH1 + TM>>64 - vpxorq %%GH1L, %%GH1L, %%GH1M ; TL = TL1 + TM<<64 -%endif + VHPXORI4x128 %%THH1, %%TLL1 %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES round 9 @@ -1909,23 +1990,10 @@ %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 - ;; ================================================= - ;; horizontal xor of low and high 4x128 -%if do_reduction != 0 - VHPXORI4x128 %%GH1H, %%GH2H - VHPXORI4x128 %%GH1L, %%GH2L -%endif - %if (NROUNDS >= 11) vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 11)] %endif - ;; ================================================= - ;; first phase of reduction -%if do_reduction != 0 - vpclmulqdq XWORD(%%RED_P1), XWORD(%%RED_POLY), XWORD(%%GH1L), 0x01 - vpslldq XWORD(%%RED_P1), XWORD(%%RED_P1), 8 ; shift-L 2 DWs - vpxorq XWORD(%%RED_P1), XWORD(%%GH1L), XWORD(%%RED_P1) ; first phase of the reduct -%endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES rounds up to 11 (AES192) or 13 (AES256) ;; AES128 is done @@ -1956,17 +2024,6 @@ %endif ; GCM256 / NROUNDS = 13 (15 including the first and the last) %endif ; GCM192 / NROUNDS = 11 (13 including the first and the last) - ;; ================================================= - ;; second phase of the reduction -%if do_reduction != 0 - vpclmulqdq XWORD(%%RED_T1), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x00 - vpsrldq XWORD(%%RED_T1), XWORD(%%RED_T1), 4 ; shift-R 1-DW to obtain 2-DWs shift-R - - vpclmulqdq XWORD(%%RED_T2), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x10 - vpslldq XWORD(%%RED_T2), XWORD(%%RED_T2), 4 ; shift-L 1-DW for result without shifts - ;; GH1H = GH1H + RED_T1 + RED_T2 - vpternlogq XWORD(%%GH1H), XWORD(%%RED_T2), XWORD(%%RED_T1), 0x96 -%endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; the last AES round ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenclast, \ @@ -1980,6 +2037,7 @@ %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ %%DATA1, %%DATA2, %%DATA3, %%DATA4 + ;; **B00_03, B04_07, B08_11, B12_15 may contain plain text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; retrieve the last cipher counter block (partially XOR'ed with text) @@ -2052,17 +2110,19 @@ %endif %if do_reduction != 0 - ;; GH1H holds reduced hash value - ;; - normally do "vmovdqa64 XWORD(%%HASH_IN_OUT), XWORD(%%GH1H)" + ;; THH1 holds reduced hash value + ;; - normally do "vmovdqa64 XWORD(%%HASH_IN_OUT), XWORD(%%THH1)" ;; - register rename trick obsoletes the above move %endif ;; ================================================= ;; GHASH last N blocks ;; - current hash value in HASH_IN_OUT or - ;; product parts in TO_REDUCE_H/M/L + ;; product parts in TO_REDUCE_H/L ;; - DATA1-DATA4 include blocks for GHASH + ;; **AESKEY1 and AESKEY2 contain AES round keys + %if do_reduction == 0 INITIAL_BLOCKS_PARTIAL_GHASH \ %%GDATA, %%GCTX, %%LENGTH, \ @@ -2070,8 +2130,8 @@ %%INSTANCE_TYPE, %%DATA1, %%DATA2, %%DATA3, %%DATA4, \ XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ %%B00_03, %%B04_07, %%B08_11, %%B12_15, %%GHDAT1, %%GHDAT2, \ - %%AESKEY1, %%AESKEY2, %%GHKEY1, \ - %%TO_REDUCE_H, %%TO_REDUCE_M, %%TO_REDUCE_L + %%AESKEY1, %%AESKEY2, %%GHKEY1, %%GHKEY2, \ + %%TO_REDUCE_H, %%TO_REDUCE_L %else INITIAL_BLOCKS_PARTIAL_GHASH \ %%GDATA, %%GCTX, %%LENGTH, \ @@ -2079,8 +2139,18 @@ %%INSTANCE_TYPE, %%DATA1, %%DATA2, %%DATA3, %%DATA4, \ XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ %%B00_03, %%B04_07, %%B08_11, %%B12_15, %%GHDAT1, %%GHDAT2, \ - %%AESKEY1, %%AESKEY2, %%GHKEY1 + %%AESKEY1, %%AESKEY2, %%GHKEY1, %%GHKEY2 +%endif + ;; **B04_07 may not get cleared (message below 4 blocks) and it may still contain sensitive data + + ;; if number of blocks is 4 then AESKEY1 and AESKEY2 do not get cleared in the macro above +%ifdef SAFE_DATA +%if %%NUM_BLOCKS == 4 + vpxorq %%AESKEY1, %%AESKEY1, %%AESKEY1 + vpxorq %%AESKEY2, %%AESKEY2, %%AESKEY2 +%endif %endif + %endmacro ;;; =========================================================================== @@ -2100,7 +2170,7 @@ %define %%GHASHIN_BLK_OFFSET %10 ; [in] numerical offset for GHASH blocks in %define %%SHFMSK %11 ; [in] ZMM with byte swap mask for pshufb %define %%ZT00 %12 ; [clobbered] temporary ZMM -%define %%ZT01 %13 ; [clobbered] temporary ZMM +%define %%ZT01 %13 ; [clobbered**] temporary ZMM %define %%ZT02 %14 ; [clobbered] temporary ZMM %define %%ZT03 %15 ; [clobbered] temporary ZMM %define %%ZT04 %16 ; [clobbered] temporary ZMM @@ -2122,12 +2192,12 @@ %define %%ZT20 %32 ; [clobbered] temporary ZMM %define %%ZT21 %33 ; [clobbered] temporary ZMM %define %%ZT22 %34 ; [clobbered] temporary ZMM -%define %%ADDBE_4x4 %35 ; [in] ZMM with 4x128bits 4 in big-endian -%define %%ADDBE_1234 %36 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian -%define %%GHASH_TYPE %37 ; [in] "start", "start_reduce", "mid", "end_reduce" -%define %%TO_REDUCE_L %38 ; [in] ZMM for low 4x128-bit GHASH sum -%define %%TO_REDUCE_H %39 ; [in] ZMM for hi 4x128-bit GHASH sum -%define %%TO_REDUCE_M %40 ; [in] ZMM for medium 4x128-bit GHASH sum +%define %%ZT23 %35 ; [clobbered] temporary ZMM +%define %%ADDBE_4x4 %36 ; [in] ZMM with 4x128bits 4 in big-endian +%define %%ADDBE_1234 %37 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian +%define %%GHASH_TYPE %38 ; [in] "start", "start_reduce", "mid", "end_reduce" +%define %%TO_REDUCE_L %39 ; [in] ZMM for low 4x128-bit GHASH sum +%define %%TO_REDUCE_H %40 ; [in] ZMM for hi 4x128-bit GHASH sum %define %%ENC_DEC %41 ; [in] cipher direction %define %%HASH_IN_OUT %42 ; [in/out] XMM ghash in/out value %define %%IA0 %43 ; [clobbered] GP temporary @@ -2191,12 +2261,12 @@ %%HASHKEY_OFFSET, %%GHASHIN_BLK_OFFSET, %%SHFMSK, \ %%ZT00, %%ZT01, %%ZT02, %%ZT03, %%ZT04, %%ZT05, %%ZT06, %%ZT07, \ %%ZT08, %%ZT09, %%ZT10, %%ZT11, %%ZT12, %%ZT13, %%ZT14, %%ZT15, \ - %%ZT16, %%ZT17, %%ZT18, %%ZT19, %%ZT20, %%ZT21, %%ZT22, \ + %%ZT16, %%ZT17, %%ZT18, %%ZT19, %%ZT20, %%ZT21, %%ZT22, %%ZT23, \ %%ADDBE_4x4, %%ADDBE_1234, %%GHASH_TYPE, \ - %%TO_REDUCE_L, %%TO_REDUCE_H, %%TO_REDUCE_M, \ + %%TO_REDUCE_L, %%TO_REDUCE_H, \ %%ENC_DEC, %%HASH_IN_OUT, %%IA0, %%IA1, %%MASKREG, \ num_blocks, %%INSTANCE_TYPE - + ;; **ZT01 may contain sensitive data jmp %%_last_blocks_done %assign num_blocks (num_blocks + 1) %endrep @@ -2212,10 +2282,11 @@ %xdefine %%GHASH_TYPE start_reduce %endif - GHASH_16 %%GHASH_TYPE, %%TO_REDUCE_H, %%TO_REDUCE_M, %%TO_REDUCE_L, \ + GHASH_16 %%GHASH_TYPE, hk_load, %%TO_REDUCE_H, %%TO_REDUCE_L, \ rsp, %%GHASHIN_BLK_OFFSET, 0, %%GDATA, %%HASHKEY_OFFSET, 0, %%HASH_IN_OUT, \ - %%ZT00, %%ZT01, %%ZT02, %%ZT03, %%ZT04, %%ZT05, %%ZT06, %%ZT07, %%ZT08, %%ZT09 - + %%ZT00, %%ZT09, %%ZT02, %%ZT03, %%ZT04, %%ZT05, %%ZT06, %%ZT07, \ + %%ZT08, %%ZT01, %%ZT23 + ;; **ZT01 may include sensitive data %%_last_blocks_done: %endmacro @@ -2253,8 +2324,8 @@ %define %%ZT14 %24 ; [clobbered] temporary ZMM (ghash) %define %%ZT15 %25 ; [clobbered] temporary ZMM (ghash) %define %%ZT16 %26 ; [clobbered] temporary ZMM (ghash) -%define %%ZT17 %27 ; [clobbered] temporary ZMM (ghash) -%define %%ZT18 %28 ; [clobbered] temporary ZMM (ghash) +%define %%ZT17 %27 ; [clobbered**] temporary ZMM (ghash) +%define %%ZT18 %28 ; [clobbered**] temporary ZMM (ghash) %define %%ZT19 %29 ; [clobbered] temporary ZMM %define %%ZT20 %30 ; [clobbered] temporary ZMM %define %%ZT21 %31 ; [clobbered] temporary ZMM @@ -2264,36 +2335,32 @@ %define %%ADDBE_1234 %35 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian %define %%TO_REDUCE_L %36 ; [in/out] ZMM for low 4x128-bit GHASH sum %define %%TO_REDUCE_H %37 ; [in/out] ZMM for hi 4x128-bit GHASH sum -%define %%TO_REDUCE_M %38 ; [in/out] ZMM for medium 4x128-bit GHASH sum -%define %%DO_REDUCTION %39 ; [in] "no_reduction", "final_reduction", "first_time" -%define %%ENC_DEC %40 ; [in] cipher direction -%define %%DATA_DISPL %41 ; [in] fixed numerical data displacement/offset -%define %%GHASH_IN %42 ; [in] current GHASH value or "no_ghash_in" +%define %%DO_REDUCTION %38 ; [in] "no_reduction", "final_reduction", "first_time" or + ; "final_reduction_no_hxor" +%define %%ENC_DEC %39 ; [in] cipher direction +%define %%DATA_DISPL %40 ; [in] fixed numerical data displacement/offset +%define %%GHASH_IN %41 ; [in] current GHASH value or "no_ghash_in" +%define %%HK_LOAD_TYPE %42 ; [in] hash key load type: hk_load or hk_bcast %define %%B00_03 %%ZT1 %define %%B04_07 %%ZT2 %define %%B08_11 %%ZT3 %define %%B12_15 %%ZT4 -%define %%GH1H %%ZT5 ; @note: do not change this mapping -%define %%GH1L %%ZT6 -%define %%GH1M %%ZT7 -%define %%GH1T %%ZT8 +%define %%THH1 %%ZT5 ; @note: do not change this mapping +%define %%THL1 %%ZT6 +%define %%TLH1 %%ZT7 +%define %%TLL1 %%ZT8 -%define %%GH2H %%ZT9 -%define %%GH2L %%ZT10 -%define %%GH2M %%ZT11 -%define %%GH2T %%ZT12 +%define %%THH2 %%ZT9 +%define %%THL2 %%ZT10 +%define %%TLH2 %%ZT11 +%define %%TLL2 %%ZT12 -%define %%RED_POLY %%GH2T -%define %%RED_P1 %%GH2L -%define %%RED_T1 %%GH2H -%define %%RED_T2 %%GH2M - -%define %%GH3H %%ZT13 -%define %%GH3L %%ZT14 -%define %%GH3M %%ZT15 -%define %%GH3T %%ZT16 +%define %%THH3 %%ZT13 +%define %%THL3 %%ZT14 +%define %%TLH3 %%ZT15 +%define %%TLL3 %%ZT16 %define %%DATA1 %%ZT13 %define %%DATA2 %%ZT14 @@ -2308,6 +2375,28 @@ %define %%GHDAT1 %%ZT21 %define %%GHDAT2 %%ZT22 +%assign hk_broadcast 0 ; normal load +%assign do_hash_reduction 0 ; no reduction +%assign do_hash_hxor 0 ; no hxor on reduced hash +%assign is_hash_start 0 ; continue with hash + +%ifidn %%HK_LOAD_TYPE, hk_bcast +%assign hk_broadcast 1 +%endif + +%ifidn %%DO_REDUCTION, final_reduction +%assign do_hash_reduction 1 +%assign do_hash_hxor 1 +%endif + +%ifidn %%DO_REDUCTION, final_reduction_no_hxor +%assign do_hash_reduction 1 +%endif + +%ifidn %%DO_REDUCTION, first_time +%assign is_hash_start 1 +%endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; prepare counter blocks @@ -2339,7 +2428,13 @@ %else vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] %endif +%if hk_broadcast != 0 + vbroadcastf64x2 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (0*64)] + vbroadcastf64x2 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (0*64) + HKeyGap] +%else vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (0*64)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (0*64) + HKeyGap] +%endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; save counter for the next round @@ -2350,7 +2445,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; pre-load constants vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 1)] - vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (1*64)] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2367,12 +2461,18 @@ ;;================================================== ;; GHASH 4 blocks (15 to 12) - vpclmulqdq %%GH1H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 - vpclmulqdq %%GH1L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 - vpclmulqdq %%GH1M, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 - vpclmulqdq %%GH1T, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 - - vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (2*64)] + vpclmulqdq %%TLL1, %%GHDAT1, %%GHKEY2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH1, %%GHDAT1, %%GHKEY2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL1, %%GHDAT1, %%GHKEY1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH1, %%GHDAT1, %%GHKEY1, 0x11 ; THH = MH*HH + +%if hk_broadcast != 0 + vbroadcastf64x2 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] + vbroadcastf64x2 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (1*64) + HKeyGap] +%else + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (1*64) + HKeyGap] +%endif vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (2*64)] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2385,12 +2485,18 @@ ;; ================================================= ;; GHASH 4 blocks (11 to 8) - vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 - vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 - vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 - vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 - - vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vpclmulqdq %%TLL2, %%GHDAT2, %%GHKEY2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%GHDAT2, %%GHKEY2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%GHDAT2, %%GHKEY1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%GHDAT2, %%GHKEY1, 0x11 ; THH = MH*HH + +%if hk_broadcast != 0 + vbroadcastf64x2 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (2*64)] + vbroadcastf64x2 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (2*64) + HKeyGap] +%else + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (2*64)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (2*64) + HKeyGap] +%endif vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (3*64)] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2403,10 +2509,18 @@ ;; ================================================= ;; GHASH 4 blocks (7 to 4) - vpclmulqdq %%GH3M, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 - vpclmulqdq %%GH3T, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 - vpclmulqdq %%GH3H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 - vpclmulqdq %%GH3L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + vpclmulqdq %%TLL3, %%GHDAT1, %%GHKEY2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH3, %%GHDAT1, %%GHKEY2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL3, %%GHDAT1, %%GHKEY1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH3, %%GHDAT1, %%GHKEY1, 0x11 ; THH = MH*HH + +%if hk_broadcast != 0 + vbroadcastf64x2 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vbroadcastf64x2 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (3*64) + HKeyGap] +%else + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (3*64) + HKeyGap] +%endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES rounds 3 @@ -2418,10 +2532,10 @@ ;; ================================================= ;; Gather (XOR) GHASH for 12 blocks - vpternlogq %%GH1H, %%GH2H, %%GH3H, 0x96 - vpternlogq %%GH1L, %%GH2L, %%GH3L, 0x96 - vpternlogq %%GH1T, %%GH2T, %%GH3T, 0x96 - vpternlogq %%GH1M, %%GH2M, %%GH3M, 0x96 + vpternlogq %%TLL1, %%TLL2, %%TLL3, 0x96 + vpternlogq %%TLH1, %%TLH2, %%TLH3, 0x96 + vpternlogq %%THL1, %%THL2, %%THL3, 0x96 + vpternlogq %%THH1, %%THH2, %%THH3, 0x96 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES rounds 4 @@ -2448,10 +2562,10 @@ ;; ================================================= ;; GHASH 4 blocks (3 to 0) - vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 - vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 - vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 - vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + vpclmulqdq %%TLL2, %%GHDAT2, %%GHKEY2, 0x00 ; TLL = ML*KL + vpclmulqdq %%TLH2, %%GHDAT2, %%GHKEY2, 0x10 ; TLH = ML*KH + vpclmulqdq %%THL2, %%GHDAT2, %%GHKEY1, 0x01 ; THL = MH*HL + vpclmulqdq %%THH2, %%GHDAT2, %%GHKEY1, 0x11 ; THH = MH*HH ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES round 6 @@ -2462,29 +2576,18 @@ vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 8)] ;; ================================================= - ;; gather GHASH in GH1L (low) and GH1H (high) -%ifidn %%DO_REDUCTION, first_time - vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM - vpxorq %%TO_REDUCE_M, %%GH1M, %%GH2M ; TM - vpxorq %%TO_REDUCE_H, %%GH1H, %%GH2H ; TH - vpxorq %%TO_REDUCE_L, %%GH1L, %%GH2L ; TL -%endif -%ifidn %%DO_REDUCTION, no_reduction - vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM - vpternlogq %%TO_REDUCE_M, %%GH1M, %%GH2M, 0x96 ; TM - vpternlogq %%TO_REDUCE_H, %%GH1H, %%GH2H, 0x96 ; TH - vpternlogq %%TO_REDUCE_L, %%GH1L, %%GH2L, 0x96 ; TL -%endif -%ifidn %%DO_REDUCTION, final_reduction - ;; phase 1: add mid products together - ;; also load polynomial constant for reduction - vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM - vpternlogq %%GH1M, %%TO_REDUCE_M, %%GH2M, 0x96 - - vpsrldq %%GH2M, %%GH1M, 8 - vpslldq %%GH1M, %%GH1M, 8 - - vmovdqa64 XWORD(%%RED_POLY), [rel POLY2] + ;; gather GHASH in TO_REDUCE_H/L +%if is_hash_start != 0 + vpxorq %%TO_REDUCE_L, %%TLL2, %%THL2 + vpxorq %%TO_REDUCE_H, %%THH2, %%TLH2 + vpternlogq %%TO_REDUCE_L, %%TLL1, %%THL1, 0x96 + vpternlogq %%TO_REDUCE_H, %%THH1, %%TLH1, 0x96 +%else + ;; not the first round so sums need to be updated + vpternlogq %%TO_REDUCE_L, %%TLL2, %%THL2, 0x96 + vpternlogq %%TO_REDUCE_H, %%THH2, %%TLH2, 0x96 + vpternlogq %%TO_REDUCE_L, %%TLL1, %%THL1, 0x96 + vpternlogq %%TO_REDUCE_H, %%THH1, %%TLH1, 0x96 %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2496,12 +2599,11 @@ vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 9)] ;; ================================================= - ;; Add mid product to high and low -%ifidn %%DO_REDUCTION, final_reduction - vpternlogq %%GH1H, %%GH2H, %%GH2M, 0x96 ; TH = TH1 + TH2 + TM>>64 - vpxorq %%GH1H, %%TO_REDUCE_H - vpternlogq %%GH1L, %%GH2L, %%GH1M, 0x96 ; TL = TL1 + TL2 + TM<<64 - vpxorq %%GH1L, %%TO_REDUCE_L + ;; new reduction +%if do_hash_reduction != 0 + vpclmulqdq %%THH1, %%TO_REDUCE_L, [rel POLY], 0x10 + vpshufd %%TO_REDUCE_L, %%TO_REDUCE_L, 01001110b + vpternlogq %%THH1, %%TO_REDUCE_H, %%TO_REDUCE_L, 0x96 %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2513,10 +2615,9 @@ vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 10)] ;; ================================================= - ;; horizontal xor of low and high 4x128 -%ifidn %%DO_REDUCTION, final_reduction - VHPXORI4x128 %%GH1H, %%GH2H - VHPXORI4x128 %%GH1L, %%GH2L + ;; horizontal xor of 4 reduced hashes +%if do_hash_hxor != 0 + VHPXORI4x128 %%THH1, %%TLL1 %endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2528,13 +2629,6 @@ %if (NROUNDS >= 11) vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 11)] %endif - ;; ================================================= - ;; first phase of reduction -%ifidn %%DO_REDUCTION, final_reduction - vpclmulqdq XWORD(%%RED_P1), XWORD(%%RED_POLY), XWORD(%%GH1L), 0x01 - vpslldq XWORD(%%RED_P1), XWORD(%%RED_P1), 8 ; shift-L 2 DWs - vpxorq XWORD(%%RED_P1), XWORD(%%GH1L), XWORD(%%RED_P1) ; first phase of the reduct -%endif ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; AES rounds up to 11 (AES192) or 13 (AES256) @@ -2566,24 +2660,13 @@ %endif ; GCM256 / NROUNDS = 13 (15 including the first and the last) %endif ; GCM192 / NROUNDS = 11 (13 including the first and the last) - ;; ================================================= - ;; second phase of the reduction -%ifidn %%DO_REDUCTION, final_reduction - vpclmulqdq XWORD(%%RED_T1), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x00 - vpsrldq XWORD(%%RED_T1), XWORD(%%RED_T1), 4 ; shift-R 1-DW to obtain 2-DWs shift-R - - vpclmulqdq XWORD(%%RED_T2), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x10 - vpslldq XWORD(%%RED_T2), XWORD(%%RED_T2), 4 ; shift-L 1-DW for result without shifts - ;; GH1H = GH1H x RED_T1 x RED_T2 - vpternlogq XWORD(%%GH1H), XWORD(%%RED_T2), XWORD(%%RED_T1), 0x96 -%endif - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; the last AES round vaesenclast %%B00_03, %%B00_03, %%AESKEY1 vaesenclast %%B04_07, %%B04_07, %%AESKEY1 vaesenclast %%B08_11, %%B08_11, %%AESKEY1 vaesenclast %%B12_15, %%B12_15, %%AESKEY1 + ;; **AESKEY1 and AESKEY2 contain AES round keys ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; XOR against plain/cipher text @@ -2598,6 +2681,7 @@ VX512STR [%%CIPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (1 * 64)], %%B04_07 VX512STR [%%CIPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (2 * 64)], %%B08_11 VX512STR [%%CIPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (3 * 64)], %%B12_15 + ;; **B00_03, B04_07, B08_011, B12_B15 may contain sensitive data ;; ================================================= ;; shuffle cipher text blocks for GHASH computation @@ -2612,6 +2696,7 @@ vpshufb %%B08_11, %%DATA3, %%SHFMSK vpshufb %%B12_15, %%DATA4, %%SHFMSK %endif + ;; **B00_03, B04_07, B08_011, B12_B15 overwritten with shuffled cipher text ;; ================================================= ;; store shuffled cipher text for ghashing @@ -2620,9 +2705,9 @@ vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (2*64)], %%B08_11 vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (3*64)], %%B12_15 -%ifidn %%DO_REDUCTION, final_reduction +%if do_hash_reduction != 0 ;; ================================================= - ;; Return GHASH value through %%GH1H + ;; Return GHASH value through %THH1 %endif %endmacro ; GHASH_16_ENCRYPT_16_PARALLEL @@ -2644,20 +2729,31 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Save register content for the caller -%macro FUNC_SAVE 0 +%macro FUNC_SAVE 0-1 + +%assign my_frame_size (STACK_FRAME_SIZE) + +%if %0 > 0 +%ifidni %1, small_frame +%assign my_frame_size (STACK_FRAME_SIZE_SMALL) +%endif + +%ifidni %1, alloc_context +%assign my_frame_size (my_frame_size + CONTEXT_SIZE) +%endif +%endif ;; Required for Update/GMC_ENC - ;the number of pushes must equal STACK_OFFSET mov rax, rsp - sub rsp, STACK_FRAME_SIZE + sub rsp, my_frame_size and rsp, ~63 mov [rsp + STACK_GP_OFFSET + 0*8], r12 mov [rsp + STACK_GP_OFFSET + 1*8], r13 mov [rsp + STACK_GP_OFFSET + 2*8], r14 mov [rsp + STACK_GP_OFFSET + 3*8], r15 - mov [rsp + STACK_GP_OFFSET + 4*8], rax ; stack - mov r14, rax ; r14 is used to retrieve stack args + mov [rsp + STACK_GP_OFFSET + 4*8], rax ; stack + mov r14, rax ; r14 is used to retrieve stack args mov [rsp + STACK_GP_OFFSET + 5*8], rbp mov [rsp + STACK_GP_OFFSET + 6*8], rbx %ifidn __OUTPUT_FORMAT__, win64 @@ -2684,12 +2780,7 @@ ;;; Restore register content for the caller %macro FUNC_RESTORE 0 -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_scratch_zmms_asm -%else vzeroupper -%endif %ifidn __OUTPUT_FORMAT__, win64 vmovdqu xmm15, [rsp + STACK_XMM_OFFSET + 9*16] @@ -2704,7 +2795,7 @@ vmovdqu xmm6, [rsp + STACK_XMM_OFFSET + 0*16] %endif - ;; Required for Update/GMC_ENC + ;; Required for Update/GCM_ENC mov rbp, [rsp + STACK_GP_OFFSET + 5*8] mov rbx, [rsp + STACK_GP_OFFSET + 6*8] %ifidn __OUTPUT_FORMAT__, win64 @@ -2757,6 +2848,7 @@ %%ZT4, %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, %%ZT10, %%ZT11, \ %%ZT12, %%ZT13, %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ %%T1, %%T2, %%T3, %%MASKREG + ;; **ZT1, ZT2, ZT3 may contain sensitive data ;; Calculate GHASH of last 16-byte block (0 || len(IV)64) mov %%T1, %%IV_LEN @@ -2767,7 +2859,9 @@ vpxorq %%J0, XWORD(%%ZT2), %%J0 vmovdqu64 XWORD(%%ZT0), [%%KEY + HashKey_1] - GHASH_MUL %%J0, XWORD(%%ZT0), XWORD(%%ZT1), XWORD(%%ZT2), XWORD(%%ZT3), XWORD(%%ZT4), XWORD(%%ZT5) + vmovdqu64 XWORD(%%ZT5), [%%KEY + HashKey_1 + HKeyGap] + GHASH_MUL2 %%J0, XWORD(%%ZT0), XWORD(%%ZT5), XWORD(%%ZT1), XWORD(%%ZT2), XWORD(%%ZT3), XWORD(%%ZT4) + ;; **ZT1, ZT2, ZT3 overwritten with ghash products vpshufb %%J0, [rel SHUF_MASK] ; perform a 16Byte swap %endmacro @@ -2838,11 +2932,12 @@ mov %%GPR1, %%A_IN vmovdqu8 XWORD(%%AAD_HASH){%%MASKREG}{z}, [%%GPR1] vmovdqu8 XWORD(%%ZT0), [%%GDATA_KEY + HashKey_1] + vmovdqu8 XWORD(%%ZT5), [%%GDATA_KEY + HashKey_1 + HKeyGap] vpshufb XWORD(%%AAD_HASH), [rel SHUF_MASK] ;; GHASH 12 bytes of AAD - GHASH_MUL XWORD(%%AAD_HASH), XWORD(%%ZT0), \ - XWORD(%%ZT1), XWORD(%%ZT2), XWORD(%%ZT3), XWORD(%%ZT4), XWORD(%%ZT5) + GHASH_MUL2 XWORD(%%AAD_HASH), XWORD(%%ZT0), XWORD(%%ZT5), \ + XWORD(%%ZT1), XWORD(%%ZT2), XWORD(%%ZT3), XWORD(%%ZT4) jmp %%_aad_compute_done @@ -2852,6 +2947,7 @@ %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, \ %%ZT10, %%ZT11, %%ZT12, %%ZT13, %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ %%GPR1, %%GPR2, %%GPR3, %%MASKREG + ;; **ZT1, ZT2, ZT3 may contain AAD but AAD is not considered sensitive %%_aad_compute_done: ;; set up context fields @@ -2889,16 +2985,16 @@ %define %%HASH_IN_OUT %11 ; [in/out] XMM GHASH value %define %%INSTANCE_TYPE %12 ; [in] single or multi call %define %%ZTMP0 %13 ; [clobbered] ZMM register -%define %%ZTMP1 %14 ; [clobbered] ZMM register -%define %%ZTMP2 %15 ; [clobbered] ZMM register -%define %%ZTMP3 %16 ; [clobbered] ZMM register +%define %%ZTMP1 %14 ; [clobbered**] ZMM register +%define %%ZTMP2 %15 ; [clobbered**] ZMM register +%define %%ZTMP3 %16 ; [clobbered**] ZMM register %define %%ZTMP4 %17 ; [clobbered] ZMM register %define %%ZTMP5 %18 ; [clobbered] ZMM register %define %%ZTMP6 %19 ; [clobbered] ZMM register %define %%ZTMP7 %20 ; [clobbered] ZMM register %define %%ZTMP8 %21 ; [clobbered] ZMM register %define %%ZTMP9 %22 ; [clobbered] ZMM register -%define %%ZTMP10 %23 ; [clobbered] ZMM register +%define %%ZTMP10 %23 ; [clobbered**] ZMM register %define %%ZTMP11 %24 ; [clobbered] ZMM register %define %%ZTMP12 %25 ; [clobbered] ZMM register %define %%ZTMP13 %26 ; [clobbered] ZMM register @@ -2968,8 +3064,9 @@ %%CTR, %%HASH_IN_OUT, %%ENC_DEC, %%INSTANCE_TYPE, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, \ - %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, %%ZTMP14, \ + %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ %%IA0, %%IA1, %%MASKREG, %%SHUFMASK + ;; **ZTMP1, ZTMP2, ZTMP3, ZTMP10 may contain sensitive data %if num_blocks != 16 jmp %%_small_initial_blocks_encrypted %endif @@ -3020,9 +3117,9 @@ %define %%AAD_HASHx xmm14 ; hardcoded in GCM_COMPLETE %define %%ZTMP0 zmm0 -%define %%ZTMP1 zmm3 -%define %%ZTMP2 zmm4 -%define %%ZTMP3 zmm5 +%define %%ZTMP1 zmm3 ; **sensitive +%define %%ZTMP2 zmm4 ; **sensitive (small data) +%define %%ZTMP3 zmm5 ; **sensitive (small data) %define %%ZTMP4 zmm6 %define %%ZTMP5 zmm7 %define %%ZTMP6 zmm10 @@ -3043,10 +3140,11 @@ %define %%ZTMP20 zmm8 %define %%ZTMP21 zmm22 %define %%ZTMP22 zmm23 +%define %%ZTMP23 zmm26 %define %%GH zmm24 %define %%GL zmm25 -%define %%GM zmm26 + %define %%SHUF_MASK zmm29 ;;; Unused in the small packet path @@ -3055,11 +3153,6 @@ %define %%MASKREG k1 -;; reduction every 48 blocks, depth 32 blocks -;; @note 48 blocks is the maximum capacity of the stack frame -%assign big_loop_nblocks 48 -%assign big_loop_depth 32 - ;;; Macro flow depending on packet size ;;; - LENGTH <= 16 blocks ;;; - cipher followed by hashing (reduction) @@ -3113,6 +3206,7 @@ %%PLAIN_CIPH_LEN, %%DATA_OFFSET, %%AAD_HASHx, %%ENC_DEC, \ %%IA0, %%IA1, %%IA2, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, %%MASKREG + ;; **ZTMP1 may contain sensitive data %else xor %%DATA_OFFSET, %%DATA_OFFSET %endif @@ -3188,32 +3282,17 @@ jb %%_no_more_big_nblocks ;; ==== - ;; ==== AES-CTR + GHASH - 48 blocks loop + ;; ==== AES-CTR + GHASH - big loop with that uses GHASH limbs (no horizontal XOR applied) ;; ==== -%%_encrypt_big_nblocks: +%%_encrypt_big_nblocks_no_hxor: + cmp %%LENGTH, (2 * big_loop_nblocks * 16) + jb %%_encrypt_big_nblocks + ;; ==== AES-CTR + GHASH - 16 blocks, start -%assign aesout_offset (STACK_LOCAL_OFFSET + (32 * 16)) +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) %assign data_in_out_offset (0 * 16) %assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) - GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ - %%CTR_BLOCKz, %%CTR_CHECK, \ - HashKey_48, aesout_offset, ghashin_offset, %%SHUF_MASK, \ - %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ - %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ - %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ - %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ - %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ - %%ZTMP20, %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, \ - %%GL, %%GH, %%GM, \ - first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz - - ;; ==== AES-CTR + GHASH - 16 blocks, no reduction -%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) -%assign data_in_out_offset (16 * 16) -%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) - GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ %%CTR_BLOCKz, %%CTR_CHECK, \ HashKey_32, aesout_offset, ghashin_offset, %%SHUF_MASK, \ @@ -3223,14 +3302,13 @@ %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ %%ZTMP20, %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, \ - %%GL, %%GH, %%GM, \ - no_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + %%ADDBE_4x4, %%ADDBE_1234, %%GL, %%GH, \ + first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz, hk_bcast ;; ==== AES-CTR + GHASH - 16 blocks, reduction %assign aesout_offset (STACK_LOCAL_OFFSET + (16 * 16)) -%assign data_in_out_offset (32 * 16) -%assign ghashin_offset (STACK_LOCAL_OFFSET + (32 * 16)) +%assign data_in_out_offset (16 * 16) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ %%CTR_BLOCKz, %%CTR_CHECK, \ @@ -3241,141 +3319,100 @@ %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ %%ZTMP20, %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, \ - %%GL, %%GH, %%GM, \ - final_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + %%ADDBE_4x4, %%ADDBE_1234, %%GL, %%GH, \ + final_reduction_no_hxor, %%ENC_DEC, data_in_out_offset, no_ghash_in, hk_bcast + ;; **ZTMP16 and ZTMP17 contain AES round keys ;; === xor cipher block 0 with GHASH (ZT4) vmovdqa64 %%AAD_HASHz, %%ZTMP4 add %%DATA_OFFSET, (big_loop_nblocks * 16) sub %%LENGTH, (big_loop_nblocks * 16) - cmp %%LENGTH, (big_loop_nblocks * 16) - jae %%_encrypt_big_nblocks - -%%_no_more_big_nblocks: - - cmp %%LENGTH, (32 * 16) - jae %%_encrypt_32_blocks - - cmp %%LENGTH, (16 * 16) - jae %%_encrypt_16_blocks - - ;; ===================================================== - ;; ===================================================== - ;; ==== GHASH 1 x 16 blocks - ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks - ;; ==== then GHASH N blocks -%%_encrypt_0_blocks_ghash_32: - ;; calculate offset to the right hash key -%ifidn %%INSTANCE_TYPE, multi_call - mov DWORD(%%IA0), DWORD(%%LENGTH) -%else - lea DWORD(%%IA0), [DWORD(%%LENGTH) + 15] -%endif - and DWORD(%%IA0), ~15 - mov DWORD(%%HASHK_PTR), HashKey_32 - sub DWORD(%%HASHK_PTR), DWORD(%%IA0) - - ;; ==== GHASH 32 blocks and follow with reduction - GHASH_16 start, %%GH, %%GM, %%GL, rsp, STACK_LOCAL_OFFSET, (0 * 16), %%GDATA_KEY, %%HASHK_PTR, 0, %%AAD_HASHz, \ - %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9 - - ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder -%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) - add DWORD(%%HASHK_PTR), (16 * 16) - - GCM_ENC_DEC_LAST \ - %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ - %%DATA_OFFSET, %%LENGTH, %%CTR_BLOCKz, %%CTR_CHECK, \ - %%HASHK_PTR, ghashin_offset, %%SHUF_MASK, \ - %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ - %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ - %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ - %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, mid, %%GL, %%GH, %%GM, \ - %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE + jmp %%_encrypt_big_nblocks_no_hxor -%ifidn %%INSTANCE_TYPE, multi_call - vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) -%endif - jmp %%_ghash_done - - ;; ===================================================== - ;; ===================================================== - ;; ==== GHASH & encrypt 1 x 16 blocks - ;; ==== GHASH & encrypt 1 x 16 blocks - ;; ==== GHASH 1 x 16 blocks (reduction) - ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks - ;; ==== then GHASH N blocks -%%_encrypt_32_blocks: + ;; ==== + ;; ==== AES-CTR + GHASH - code identical to big loop but here normal GHASH is applied + ;; ==== - this corrects GHASH limb values if exiting the loop above + ;; ==== +%%_encrypt_big_nblocks: ;; ==== AES-CTR + GHASH - 16 blocks, start -%assign aesout_offset (STACK_LOCAL_OFFSET + (32 * 16)) -%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) %assign data_in_out_offset (0 * 16) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ %%CTR_BLOCKz, %%CTR_CHECK, \ - HashKey_48, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + HashKey_32, aesout_offset, ghashin_offset, %%SHUF_MASK, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ %%ZTMP20, %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, \ - %%GL, %%GH, %%GM, \ - first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz + %%ADDBE_4x4, %%ADDBE_1234, %%GL, %%GH, \ + first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz, hk_load - ;; ==== AES-CTR + GHASH - 16 blocks, no reduction -%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) -%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) + ;; ==== AES-CTR + GHASH - 16 blocks, reduction +%assign aesout_offset (STACK_LOCAL_OFFSET + (16 * 16)) %assign data_in_out_offset (16 * 16) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ %%CTR_BLOCKz, %%CTR_CHECK, \ - HashKey_32, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + HashKey_16, aesout_offset, ghashin_offset, %%SHUF_MASK, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ %%ZTMP20, %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, \ - %%GL, %%GH, %%GM, \ - no_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + %%ADDBE_4x4, %%ADDBE_1234, %%GL, %%GH, \ + final_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in, hk_load - ;; ==== GHASH 16 blocks with reduction - GHASH_16 end_reduce, %%GH, %%GM, %%GL, rsp, STACK_LOCAL_OFFSET, (32 * 16), %%GDATA_KEY, HashKey_16, 0, %%AAD_HASHz, \ - %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9 + ;; **ZTMP16 and ZTMP17 contain AES round keys - ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder -%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) + ;; === xor cipher block 0 with GHASH (ZT4) + vmovdqa64 %%AAD_HASHx, XWORD(%%ZTMP4) - sub %%LENGTH, (32 * 16) - add %%DATA_OFFSET, (32 * 16) + add %%DATA_OFFSET, (big_loop_nblocks * 16) + sub %%LENGTH, (big_loop_nblocks * 16) - ;; calculate offset to the right hash key -%ifidn %%INSTANCE_TYPE, multi_call - mov DWORD(%%IA0), DWORD(%%LENGTH) -%else - lea DWORD(%%IA0), [DWORD(%%LENGTH) + 15] -%endif - and DWORD(%%IA0), ~15 - mov DWORD(%%HASHK_PTR), HashKey_16 - sub DWORD(%%HASHK_PTR), DWORD(%%IA0) + ;; fall through +%%_no_more_big_nblocks: + + cmp %%LENGTH, (16 * 16) + jae %%_encrypt_16_blocks + + ;; ===================================================== + ;; ===================================================== + ;; ==== GHASH 1 x 16 blocks + ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks + ;; ==== then GHASH N blocks +%%_encrypt_N_ghash_32_and_N_blocks: + ;; ==== GHASH 32 blocks and follow with reduction + GHASH_16 start, hk_load, %%GH, %%GL, rsp, STACK_LOCAL_OFFSET, (0 * 16), %%GDATA_KEY, HashKey_32, 0, %%AAD_HASHz, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP23 + ;; **ZTMP9 may include sensitive data + + ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder +%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) GCM_ENC_DEC_LAST \ %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ %%DATA_OFFSET, %%LENGTH, %%CTR_BLOCKz, %%CTR_CHECK, \ - %%HASHK_PTR, ghashin_offset, %%SHUF_MASK, \ + HashKey_16, ghashin_offset, %%SHUF_MASK, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ - %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, start, %%GL, %%GH, %%GM, \ + %%ZTMP21, %%ZTMP22, %%ZTMP23, \ + %%ADDBE_4x4, %%ADDBE_1234, end_reduce, %%GL, %%GH, \ %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE - + ;; **ZTMP9 clobbered but ZTMP1 may include sensitive data + ;; **ZTMP16 and ZTMP17 clobbered or cleared above + ;; **ZTMP21 may include part of cipher key +%ifdef SAFE_DATA + vpxorq %%ZTMP21, %%ZTMP21, %%ZTMP21 +%endif %ifidn %%INSTANCE_TYPE, multi_call vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) %endif @@ -3383,54 +3420,36 @@ ;; ===================================================== ;; ===================================================== - ;; ==== GHASH & encrypt 16 blocks (done before) + ;; ==== GHASH & encrypt 16 blocks ;; ==== GHASH 1 x 16 blocks ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks ;; ==== then GHASH N blocks %%_encrypt_16_blocks: ;; ==== AES-CTR + GHASH - 16 blocks, start -%assign aesout_offset (STACK_LOCAL_OFFSET + (32 * 16)) +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) %assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) %assign data_in_out_offset (0 * 16) GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ %%CTR_BLOCKz, %%CTR_CHECK, \ - HashKey_48, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + HashKey_32, aesout_offset, ghashin_offset, %%SHUF_MASK, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ %%ZTMP20, %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, \ - %%GL, %%GH, %%GM, \ - first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz - - ;; ==== GHASH 1 x 16 blocks - GHASH_16 mid, %%GH, %%GM, %%GL, rsp, STACK_LOCAL_OFFSET, (16 * 16), %%GDATA_KEY, HashKey_32, 0, no_hash_input, \ - %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9 + %%ADDBE_4x4, %%ADDBE_1234, %%GL, %%GH, \ + first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz, hk_load - ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder -%assign ghashin_offset (STACK_LOCAL_OFFSET + (32 * 16)) - - sub %%LENGTH, (16 * 16) - add %%DATA_OFFSET, (16 * 16) + ;; **ZTMP16 and ZTMP17 contain AES round keys - GCM_ENC_DEC_LAST \ - %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ - %%DATA_OFFSET, %%LENGTH, %%CTR_BLOCKz, %%CTR_CHECK, \ - HashKey_16, ghashin_offset, %%SHUF_MASK, \ - %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ - %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ - %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ - %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, end_reduce, %%GL, %%GH, %%GM, \ - %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE + ;; ==== GHASH 1 x 16 blocks + GHASH_16 end_reduce, hk_load, %%GH, %%GL, rsp, STACK_LOCAL_OFFSET, (16 * 16), %%GDATA_KEY, HashKey_16, 0, %%AAD_HASHz, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP23 + ;; **ZTMP9 may include sensitive data -%ifidn %%INSTANCE_TYPE, multi_call - vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) -%endif - jmp %%_ghash_done + ;; fall through here: handling of the rest is identical in both cases %%_message_below_32_blocks: ;; 32 > number of blocks > 16 @@ -3457,10 +3476,15 @@ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ - %%ZTMP21, %%ZTMP22, \ - %%ADDBE_4x4, %%ADDBE_1234, start, %%GL, %%GH, %%GM, \ + %%ZTMP21, %%ZTMP22, %%ZTMP23, \ + %%ADDBE_4x4, %%ADDBE_1234, start, %%GL, %%GH, \ %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE - + ;; **ZTMP9 clobbered but ZTMP1 may include sensitive data + ;; **ZTMP16 and ZTMP17 clobbered or cleared above + ;; **ZTMP21 may include part of cipher key +%ifdef SAFE_DATA + vpxorq %%ZTMP21, %%ZTMP21, %%ZTMP21 +%endif %ifidn %%INSTANCE_TYPE, multi_call vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) %endif @@ -3485,10 +3509,18 @@ %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ %%ZTMP20, %%ZTMP21, %%ZTMP22, \ %%IA0, %%IA3, %%MASKREG, %%SHUF_MASK - + ;; **ZTMP1, ZTMP2, ZTMP3, ZTMP10 may include sensitive data +%ifdef SAFE_DATA + vpxorq %%ZTMP2, %%ZTMP2, %%ZTMP2 + vpxorq %%ZTMP3, %%ZTMP3, %%ZTMP3 + vpxorq %%ZTMP10, %%ZTMP10, %%ZTMP10 +%endif ;; fall through to exit %%_ghash_done: +%ifdef SAFE_DATA + vpxorq %%ZTMP1, %%ZTMP1, %%ZTMP1 +%endif %ifidn %%INSTANCE_TYPE, multi_call ;; save the last counter block vmovdqu64 [%%GDATA_CTX + CurCount], %%CTR_BLOCKx @@ -3511,15 +3543,15 @@ %define %%CTR_CHECK %7 ; [in/out] GPR with counter overflow check %define %%ADDBE_4x4 %8 ; [in] ZMM 4x128bits with value 4 (big endian) %define %%ADDBE_1234 %9 ; [in] ZMM 4x128bits with values 1, 2, 3 & 4 (big endian) -%define %%T0 %10 ; [clobered] temporary ZMM register -%define %%T1 %11 ; [clobered] temporary ZMM register -%define %%T2 %12 ; [clobered] temporary ZMM register -%define %%T3 %13 ; [clobered] temporary ZMM register -%define %%T4 %14 ; [clobered] temporary ZMM register -%define %%T5 %15 ; [clobered] temporary ZMM register -%define %%T6 %16 ; [clobered] temporary ZMM register -%define %%T7 %17 ; [clobered] temporary ZMM register -%define %%T8 %18 ; [clobered] temporary ZMM register +%define %%T0 %10 ; [clobbered] temporary ZMM register +%define %%T1 %11 ; [clobbered] temporary ZMM register +%define %%T2 %12 ; [clobbered] temporary ZMM register +%define %%T3 %13 ; [clobbered] temporary ZMM register +%define %%T4 %14 ; [clobbered] temporary ZMM register +%define %%T5 %15 ; [clobbered] temporary ZMM register +%define %%T6 %16 ; [clobbered] temporary ZMM register +%define %%T7 %17 ; [clobbered] temporary ZMM register +%define %%T8 %18 ; [clobbered] temporary ZMM register %define %%SHUF_MASK %19 ; [in] ZMM with BE/LE shuffle mask %define %%ENC_DEC %20 ; [in] ENC (encrypt) or DEC (decrypt) selector %define %%BLK_OFFSET %21 ; [in] stack frame offset to ciphered blocks @@ -3612,6 +3644,7 @@ vpshufb %%B08_11, %%B08_11, %%SHUF_MASK vpshufb %%B12_15, %%B12_15, %%SHUF_MASK %endif + ;; **B00_03, B04_07, B08_11, B12_15 overwritten with shuffled cipher text %ifnidn %%GHASH, no_ghash ;; === xor cipher block 0 with GHASH for the next GHASH round @@ -3645,7 +3678,10 @@ ;; @note: xmm14 is hardcoded for hash input in singe_call case - vmovdqu xmm13, [%%GDATA_KEY + HashKey] + vmovdqu xmm13, [%%GDATA_KEY + HashKey_1] + vmovdqu xmm6, [%%GDATA_KEY + HashKey_1 + HKeyGap] + ;; **xmm13 and xmm6 contain authentication key + ;; Start AES as early as possible vmovdqu xmm9, [%%GDATA_CTX + OrigIV] ; xmm9 = Y0 ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Y0) @@ -3662,7 +3698,7 @@ je %%_partial_done ;; GHASH computation for the last <16 Byte block - GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 + GHASH_MUL2 xmm14, xmm13, xmm6, xmm0, xmm10, xmm11, xmm5 %%_partial_done: %endif @@ -3671,7 +3707,7 @@ vpsllq xmm15, xmm15, 3 ; convert bytes into bits vpxor xmm14, xmm15 - GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 + GHASH_MUL2 xmm14, xmm13, xmm6, xmm0, xmm10, xmm11, xmm5 vpshufb xmm14, [rel SHUF_MASK] ; perform a 16Byte swap vpxor xmm9, xmm9, xmm14 @@ -3707,7 +3743,7 @@ %ifdef SAFE_DATA ;; Clear sensitive data from context structure - vpxor xmm0, xmm0 + vpxor xmm0, xmm0, xmm0 vmovdqu [%%GDATA_CTX + AadHash], xmm0 %ifidn %%INSTANCE_TYPE, multi_call vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm0 @@ -3722,7 +3758,7 @@ ;;; Output: ;;; Updated AAD_HASH, DATA_OFFSET and GDATA_CTX ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro PARTIAL_BLOCK_GMAC 20 +%macro PARTIAL_BLOCK_GMAC 18 %define %%GDATA_KEY %1 ; [in] Key pointer %define %%GDATA_CTX %2 ; [in] context pointer %define %%PLAIN_IN %3 ; [in] input buffer @@ -3740,9 +3776,7 @@ %define %%ZTMP5 %15 ; [clobbered] ZMM temporary register %define %%ZTMP6 %16 ; [clobbered] ZMM temporary register %define %%ZTMP7 %17 ; [clobbered] ZMM temporary register -%define %%ZTMP8 %18 ; [clobbered] ZMM temporary register -%define %%ZTMP9 %19 ; [clobbered] ZMM temporary register -%define %%MASKREG %20 ; [clobbered] mask temporary register +%define %%MASKREG %18 ; [clobbered] mask temporary register %define %%XTMP0 XWORD(%%ZTMP0) %define %%XTMP1 XWORD(%%ZTMP1) @@ -3752,8 +3786,6 @@ %define %%XTMP5 XWORD(%%ZTMP5) %define %%XTMP6 XWORD(%%ZTMP6) %define %%XTMP7 XWORD(%%ZTMP7) -%define %%XTMP8 XWORD(%%ZTMP8) -%define %%XTMP9 XWORD(%%ZTMP9) %define %%LENGTH %%GPTMP0 %define %%IA0 %%GPTMP1 @@ -3766,7 +3798,8 @@ READ_SMALL_DATA_INPUT_LEN_BT16_AVX512 %%XTMP4, %%PLAIN_IN, %%PLAIN_LEN, \ %%IA0, %%IA1, %%MASKREG - vmovdqu64 %%XTMP2, [%%GDATA_KEY + HashKey] + vmovdqu64 %%XTMP2, [%%GDATA_KEY + HashKey_1] + vmovdqu64 %%XTMP7, [%%GDATA_KEY + HashKey_1 + HKeyGap] ;; adjust the shuffle mask pointer to be able to shift right %%LENGTH bytes ;; (16 - %%LENGTH) is the number of bytes in plaintext mod 16) @@ -3785,15 +3818,17 @@ ;; - mask out bottom %%LENGTH bytes of %%XTMP1 vmovdqu64 %%XTMP0, [%%IA0 + ALL_F - SHIFT_MASK] - vpand %%XTMP4, %%XTMP0 - vpshufb %%XTMP4, [rel SHUF_MASK] - vpshufb %%XTMP4, %%XTMP3 - vpxorq %%AAD_HASH, %%XTMP4 + vpand %%XTMP4, %%XTMP4, %%XTMP0 + vpshufb %%XTMP4, %%XTMP4, [rel SHUF_MASK] + vpshufb %%XTMP4, %%XTMP4, %%XTMP3 + vpxorq %%AAD_HASH, %%AAD_HASH, %%XTMP4 + ;; **XTMP4 contains plain text cmp %%IA1, 0 jl %%_partial_incomplete ;; GHASH computation for the last <16 Byte block - GHASH_MUL %%AAD_HASH, %%XTMP2, %%XTMP5, %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9 + GHASH_MUL2 %%AAD_HASH, %%XTMP2, %%XTMP7, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 + ;; **XTMP4 clobbered with temporary compute data mov qword [%%GDATA_CTX + PBlockLen], 0 diff --git a/lib/include/imb_job.asm b/lib/include/imb_job.inc similarity index 97% rename from lib/include/imb_job.asm rename to lib/include/imb_job.inc index 13be1902164cbf42d074ff452439beb5113f846a..e545b1f9f5b9e63668b7dab7423fffcd899236e7 100644 --- a/lib/include/imb_job.asm +++ b/lib/include/imb_job.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/datastruct.asm" +%include "include/datastruct.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Define constants @@ -214,3 +214,5 @@ END_FIELDS %assign _snow_v_aad_len _u + __snow_v_aad_len %assign _snow_v_reserved _u + __snow_v_reserved %assign _cbcs_next_iv _cipher_fields + __cbcs_next_iv +%assign _gcm_aad _u + __gcm_aad +%assign _gcm_aad_len _u + __gcm_aad_len diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 62d3f1b70378939cfeee7eb3b8c1389c66c085d2..bf86c01e1d4b207cd3ef22409c981359873eed26 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -166,6 +166,12 @@ typedef struct { * Structure to store the Snow3G state for 16 packets. *****************************************************************************/ typedef struct { +#ifndef __aarch64__ + void *in[16]; + void *out[16]; + void *keys[16]; + void *iv[16]; +#else /* * The type of in, keys and iv should be "const void *", to be * consistent with the type definition in ipsec_mb.h @@ -174,6 +180,7 @@ typedef struct { void *out[16]; const void *keys[16]; const void *iv[16]; +#endif uint32_t LFSR_0[16]; uint32_t LFSR_1[16]; uint32_t LFSR_2[16]; @@ -444,6 +451,8 @@ init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void +init_mb_mgr_avx2_t3_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs); @@ -465,6 +474,8 @@ get_next_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +get_next_burst_avx2_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t get_next_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -484,6 +495,8 @@ submit_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +submit_burst_avx2_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t submit_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -510,6 +523,9 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx2_t3(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -531,6 +547,8 @@ flush_burst_avx2_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx2_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +flush_burst_avx2_t3(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t flush_burst_avx512_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx512_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); @@ -578,6 +596,12 @@ submit_cipher_burst_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx2_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t submit_cipher_burst_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_CIPHER_MODE cipher, @@ -633,6 +657,12 @@ submit_cipher_burst_nocheck_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx2_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t submit_cipher_burst_nocheck_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_CIPHER_MODE cipher, @@ -674,6 +704,10 @@ submit_hash_burst_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t +submit_hash_burst_avx2_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t submit_hash_burst_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); @@ -711,6 +745,10 @@ submit_hash_burst_nocheck_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_avx2_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t submit_hash_burst_nocheck_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); @@ -775,6 +813,14 @@ IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2_t2(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2_t2(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2_t2(IMB_MGR *state); +/* AVX2 TYPE3 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx2_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_avx2_t3(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_avx2_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2_t3(IMB_MGR *state); + /* AVX512 TYPE1 manager functions */ IMB_DLL_EXPORT IMB_JOB *submit_job_avx512_t1(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *flush_job_avx512_t1(IMB_MGR *state); @@ -791,11 +837,123 @@ IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512_t2(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512_t2(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512_t2(IMB_MGR *state); +#ifdef __aarch64__ +/* noaesni functions */ IMB_DLL_LOCAL void init_mb_mgr_aarch64_no_aesni_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_EXPORT uint32_t +get_next_burst_aarch64_no_aesni(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_aarch64_no_aesni(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_aarch64_no_aesni(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_aarch64_no_aesni(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_aarch64_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT IMB_JOB *submit_job_aarch64_no_aesni(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_aarch64_no_aesni(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_aarch64_no_aesni(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_aarch64_no_aesni(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_aarch64_no_aesni(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_aarch64_no_aesni(IMB_MGR *state); + +/* aarch64 functions */ IMB_DLL_LOCAL void init_mb_mgr_aarch64_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_EXPORT uint32_t +get_next_burst_aarch64(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_aarch64(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_aarch64(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_aarch64(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_aarch64(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_aarch64(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_aarch64(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_aarch64(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT IMB_JOB *submit_job_aarch64(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_aarch64(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_aarch64(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_aarch64(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_aarch64(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_aarch64(IMB_MGR *state); + +/* sve256 functions */ IMB_DLL_LOCAL void init_mb_mgr_aarch64_sve256_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_EXPORT uint32_t +get_next_burst_aarch64_sve256(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_aarch64_sve256(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_aarch64_sve256(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_aarch64_sve256(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_aarch64_sve256(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_aarch64_sve256(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_aarch64_sve256(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_aarch64_sve256(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT IMB_JOB *submit_job_aarch64_sve256(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_aarch64_sve256(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_aarch64_sve256(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_aarch64_sve256(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_aarch64_sve256(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_aarch64_sve256(IMB_MGR *state); +#endif #endif /* IMB_IPSEC_MB_INTERNAL_H */ diff --git a/lib/include/job_api_docsis.h b/lib/include/job_api_docsis.h index 17236b01e7f811c1163ba3456eb04e453f6b8ccd..9494ab73a790a3e25a58aa7a54c461d703a19221 100644 --- a/lib/include/job_api_docsis.h +++ b/lib/include/job_api_docsis.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -33,9 +33,9 @@ __forceinline IMB_JOB * -submit_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) +submit_docsis_enc_job(IMB_MGR *state, IMB_JOB *job, const uint64_t key_sz) { - if (16 == job->key_len_in_bytes) { + if (16 == key_sz) { if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { MB_MGR_DOCSIS_AES_OOO *p_ooo = state->docsis128_crc32_sec_ooo; @@ -64,9 +64,9 @@ submit_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) __forceinline IMB_JOB * -flush_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) +flush_docsis_enc_job(IMB_MGR *state, IMB_JOB *job, const uint64_t key_sz) { - if (16 == job->key_len_in_bytes) { + if (16 == key_sz) { if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { MB_MGR_DOCSIS_AES_OOO *p_ooo = state->docsis128_crc32_sec_ooo; @@ -95,9 +95,9 @@ flush_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) __forceinline IMB_JOB * -submit_docsis_dec_job(IMB_MGR *state, IMB_JOB *job) +submit_docsis_dec_job(IMB_MGR *state, IMB_JOB *job, const uint64_t key_sz) { - if (16 == job->key_len_in_bytes) { + if (16 == key_sz) { if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { MB_MGR_DOCSIS_AES_OOO *p_ooo = state->docsis128_crc32_sec_ooo; diff --git a/lib/include/job_api_gcm.h b/lib/include/job_api_gcm.h index 9efb852b5f29e523af6fe0c006c6f8ce449b1cb4..12ae30417362649c51ea873bfcf7180a10fc5546 100644 --- a/lib/include/job_api_gcm.h +++ b/lib/include/job_api_gcm.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -32,9 +32,9 @@ __forceinline IMB_JOB * -submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job) +submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job, const uint64_t key_sz) { - switch (job->key_len_in_bytes) { + switch (key_sz) { case IMB_KEY_128_BYTES: if (job->sgl_state == IMB_SGL_INIT) IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, @@ -162,9 +162,9 @@ submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job) __forceinline IMB_JOB * -submit_gcm_sgl_dec(IMB_MGR *state, IMB_JOB *job) +submit_gcm_sgl_dec(IMB_MGR *state, IMB_JOB *job, const uint64_t key_sz) { - switch (job->key_len_in_bytes) { + switch (key_sz) { case IMB_KEY_128_BYTES: if (job->sgl_state == IMB_SGL_INIT) IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, diff --git a/lib/include/job_api_kasumi.h b/lib/include/job_api_kasumi.h index 462b3423029871ebb525007e7b32e427fb45bd80..fe2a190e2a8917fd184b2307f16ed39d46dd1ac2 100644 --- a/lib/include/job_api_kasumi.h +++ b/lib/include/job_api_kasumi.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/job_api_snowv.h b/lib/include/job_api_snowv.h index ff0dfe7d18930fabad12292bd3104022728559c5..10c6f7c3b26b740beb9d8b2ae50eef12795d5903 100644 --- a/lib/include/job_api_snowv.h +++ b/lib/include/job_api_snowv.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/kasumi_interface.h b/lib/include/kasumi_interface.h index 7ac6c5fbd5847b69738c03afd17b416b43552e90..bc03a6baac528d07bc1ad9dcb7eff65199cbbdb4 100644 --- a/lib/include/kasumi_interface.h +++ b/lib/include/kasumi_interface.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/kasumi_internal.h b/lib/include/kasumi_internal.h index 1bbb81d3f11192e2f15b6321b8be600b443db992..cd77b4efa5b56c7a1228c81a0d8e7d1332b89f46 100755 --- a/lib/include/kasumi_internal.h +++ b/lib/include/kasumi_internal.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -307,27 +307,26 @@ typedef union SafeBuffer { ******************************************************************************/ static void kasumi_1_block(const uint16_t *context, uint16_t *data) { - const uint16_t *end = context + KASUMI_KEY_SCHEDULE_SIZE; - uint16_t temp_l, temp_h; - - /* 4 iterations odd/even */ - do { - temp_l = data[3]; - temp_h = data[2]; - FLp1(context, temp_h, temp_l); - FOp1(context, temp_h, temp_l); - context += 8; - data[1] ^= temp_l; - data[0] ^= temp_h; - - temp_h = data[1]; - temp_l = data[0]; - FOp1(context, temp_h, temp_l); - FLp1(context, temp_h, temp_l); - context += 8; - data[3] ^= temp_h; - data[2] ^= temp_l; - } while (context < end); + const uint16_t *end = context + KASUMI_KEY_SCHEDULE_SIZE; + + /* 4 iterations odd/even */ + do { + uint16_t temp_l = data[3], temp_h = data[2]; + + FLp1(context, temp_h, temp_l); + FOp1(context, temp_h, temp_l); + context += 8; + data[1] ^= temp_l; + data[0] ^= temp_h; + + temp_h = data[1]; + temp_l = data[0]; + FOp1(context, temp_h, temp_l); + FLp1(context, temp_h, temp_l); + context += 8; + data[3] ^= temp_h; + data[2] ^= temp_l; + } while (context < end); } /** @@ -344,38 +343,35 @@ static void kasumi_1_block(const uint16_t *context, uint16_t *data) static void kasumi_2_blocks(const uint16_t *context, uint16_t *data1, uint16_t *data2) { - const uint16_t *end = context + KASUMI_KEY_SCHEDULE_SIZE; - uint16_t temp1_l, temp1_h; - uint16_t temp2_l, temp2_h; - - /* 4 iterations odd/even , with fine grain interleave */ - do { - /* even */ - temp1_l = data1[3]; - temp1_h = data1[2]; - temp2_l = data2[3]; - temp2_h = data2[2]; - FLp2(context, temp1_h, temp1_l, temp2_h, temp2_l); - FOp2(context, temp1_h, temp1_l, temp2_h, temp2_l); - context += 8; - data1[1] ^= temp1_l; - data1[0] ^= temp1_h; - data2[1] ^= temp2_l; - data2[0] ^= temp2_h; - - /* odd */ - temp1_h = data1[1]; - temp1_l = data1[0]; - temp2_h = data2[1]; - temp2_l = data2[0]; - FOp2(context, temp1_h, temp1_l, temp2_h, temp2_l); - FLp2(context, temp1_h, temp1_l, temp2_h, temp2_l); - context += 8; - data1[3] ^= temp1_h; - data1[2] ^= temp1_l; - data2[3] ^= temp2_h; - data2[2] ^= temp2_l; - } while (context < end); + const uint16_t *end = context + KASUMI_KEY_SCHEDULE_SIZE; + + /* 4 iterations odd/even , with fine grain interleave */ + do { + /* even */ + uint16_t temp1_l = data1[3], temp1_h = data1[2]; + uint16_t temp2_l = data2[3], temp2_h = data2[2]; + + FLp2(context, temp1_h, temp1_l, temp2_h, temp2_l); + FOp2(context, temp1_h, temp1_l, temp2_h, temp2_l); + context += 8; + data1[1] ^= temp1_l; + data1[0] ^= temp1_h; + data2[1] ^= temp2_l; + data2[0] ^= temp2_h; + + /* odd */ + temp1_h = data1[1]; + temp1_l = data1[0]; + temp2_h = data2[1]; + temp2_l = data2[0]; + FOp2(context, temp1_h, temp1_l, temp2_h, temp2_l); + FLp2(context, temp1_h, temp1_l, temp2_h, temp2_l); + context += 8; + data1[3] ^= temp1_h; + data1[2] ^= temp1_l; + data2[3] ^= temp2_h; + data2[2] ^= temp2_l; + } while (context < end); } @@ -397,18 +393,13 @@ kasumi_3_blocks(const uint16_t *context, uint16_t *data1, { /* Case when the conmpiler is able to interleave efficiently */ const uint16_t *end = context + KASUMI_KEY_SCHEDULE_SIZE; - uint16_t temp1_l, temp1_h; - uint16_t temp2_l, temp2_h; - uint16_t temp3_l, temp3_h; /* 4 iterations odd/even , with fine grain interleave */ do { - temp1_l = data1[3]; - temp1_h = data1[2]; - temp2_l = data2[3]; - temp2_h = data2[2]; - temp3_l = data3[3]; - temp3_h = data3[2]; + uint16_t temp1_l = data1[3], temp1_h = data1[2]; + uint16_t temp2_l = data2[3], temp2_h = data2[2]; + uint16_t temp3_l = data3[3], temp3_h = data3[2]; + FLp3(context, temp1_h, temp1_l, temp2_h, temp2_l, temp3_h, temp3_l); FOp3(context, temp1_h, temp1_l, temp2_h, temp2_l, temp3_h, @@ -454,9 +445,9 @@ kasumi_3_blocks(const uint16_t *context, uint16_t *data1, static void kasumi_4_blocks(const uint16_t *context, uint16_t **ppData) { - /* Case when the conmpiler is unable to interleave efficiently */ - kasumi_2_blocks (context, ppData[0], ppData[1]); - kasumi_2_blocks (context, ppData[2], ppData[3]); + /* Case when the conmpiler is unable to interleave efficiently */ + kasumi_2_blocks (context, ppData[0], ppData[1]); + kasumi_2_blocks (context, ppData[2], ppData[3]); } /** @@ -472,8 +463,8 @@ kasumi_4_blocks(const uint16_t *context, uint16_t **ppData) static void kasumi_8_blocks(const uint16_t *context, uint16_t **ppData) { - kasumi_4_blocks (context, &ppData[0]); - kasumi_4_blocks (context, &ppData[4]); + kasumi_4_blocks (context, &ppData[0]); + kasumi_4_blocks (context, &ppData[4]); } /****************************************************************************** @@ -1427,7 +1418,7 @@ kasumi_f8_n_buffer(const kasumi_key_sched_t *pKeySchedule, const uint64_t IV[], if (bufCount > 16) { pOut[0] = NULL; - printf("dataCount too high (%d)\n", bufCount); + printf("dataCount too high (%u)\n", (unsigned) bufCount); return; } @@ -1442,7 +1433,7 @@ kasumi_f8_n_buffer(const kasumi_key_sched_t *pKeySchedule, const uint64_t IV[], uint32_t blkcnt = 0; uint32_t len = 0; uint32_t packet_idx, inner_idx, same_size_blocks; - int sortNeeded = 0, tempLen = 0; + int sortNeeded = 0; SafeBuf safeInBuf = {0}; memcpy((void *)dataLen, lengths, dataCount * sizeof(uint32_t)); @@ -1491,7 +1482,8 @@ kasumi_f8_n_buffer(const kasumi_key_sched_t *pKeySchedule, const uint64_t IV[], srctempbuff = pDataIn[packet_idx]; dsttempbuff = pDataOut[packet_idx]; tempSort = temp[packet_idx]; - tempLen = dataLen[packet_idx]; + + const int tempLen = dataLen[packet_idx]; pDataIn[packet_idx] = pDataIn[inner_idx]; diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index 9186c7be0671393258379f4a7ef58462fad66d19..cb3009b78e3f3521a9e1e0d1e2bd61eb224e22eb 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -34,6 +34,7 @@ #include "include/error.h" #include "include/mb_mgr_job_check.h" /* is_job_invalid() */ +#ifndef __aarch64__ __forceinline uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, IMB_JOB *jobs, @@ -295,6 +296,7 @@ uint32_t submit_aes_ctr_burst(IMB_MGR *state, return n_jobs; } +#endif /* __aarch64__ */ __forceinline uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, @@ -314,6 +316,7 @@ uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, } switch (cipher) { +#ifndef __aarch64__ case IMB_CIPHER_CBC: if (dir == IMB_DIR_ENCRYPT) return submit_aes_cbc_burst_enc(state, jobs, n_jobs, @@ -324,6 +327,7 @@ uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, case IMB_CIPHER_CNTR: return submit_aes_ctr_burst(state, jobs, n_jobs, key_size, run_check); +#endif /* __aarch64__ */ default: break; } @@ -356,6 +360,7 @@ SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, cipher, dir, key_size, 0); } +#ifndef __aarch64__ __forceinline uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, @@ -499,6 +504,7 @@ uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, return completed_jobs; } +#endif /* __aarch64__ */ __forceinline uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, @@ -517,6 +523,7 @@ uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, } switch (hash) { +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_1); @@ -532,6 +539,7 @@ uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, case IMB_AUTH_HMAC_SHA_512: return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_512); +#endif /* __aarch64__ */ default: break; } diff --git a/lib/include/mb_mgr_burst_async.h b/lib/include/mb_mgr_burst_async.h index 5c436e3c756b21eb66e6c18ac7bca7d82e9bc51c..c9dd5bf04cabb13360c42bbc558f88b6a5c7168c 100644 --- a/lib/include/mb_mgr_burst_async.h +++ b/lib/include/mb_mgr_burst_async.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -156,6 +156,19 @@ submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, jobs[i]->key_len_in_bytes)) { goto return_invalid_job; } + + /* validate job->suite_id */ + void *t[4]; + + set_cipher_suite_id(jobs[i], t); + + if(jobs[i]->suite_id[0] != t[0] || + jobs[i]->suite_id[1] != t[1] || + jobs[i]->suite_id[2] != t[2] || + jobs[i]->suite_id[3] != t[3]) { + imb_set_errno(state, IMB_ERR_BURST_SUITE_ID); + goto return_invalid_job; + } } } @@ -166,7 +179,7 @@ submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, /* submit all jobs */ for (i = 0; i < n_jobs; i++) { jobs[i]->status = IMB_STATUS_BEING_PROCESSED; - submit_new_job(state, jobs[i]); + submit_new_burst_job(state, jobs[i]); } ADV_N_JOBS(&state->next_job, n_jobs); @@ -253,7 +266,7 @@ FLUSH_BURST(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs) IMB_JOB *job = JOBS(state, state->earliest_job); if (job->status < IMB_STATUS_COMPLETED) - complete_job(state, job); + complete_burst_job(state, job); jobs[n_ret_jobs++] = job; ADV_JOBS(&state->earliest_job); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index d5e1107152be68c102f7f0e705755b07da47f456..ef1362decb27df6ec49a0d1e9365c5cb7ce0fced 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -52,7 +52,8 @@ void ADV_JOBS(int *ptr) __forceinline uint32_t get_queue_sz(IMB_MGR *state) { - const int a = (state->next_job - state->earliest_job) / sizeof(IMB_JOB); + const int a = (state->next_job - state->earliest_job) / + (int) sizeof(IMB_JOB); return a & (IMB_MAX_JOBS-1); } diff --git a/lib/include/mb_mgr_datastruct.asm b/lib/include/mb_mgr_datastruct.inc similarity index 98% rename from lib/include/mb_mgr_datastruct.asm rename to lib/include/mb_mgr_datastruct.inc index 162820468a2731781647a046ff3eb5954a78edbc..da06c842c0936e01f230b10e660ef92903b11003 100644 --- a/lib/include/mb_mgr_datastruct.asm +++ b/lib/include/mb_mgr_datastruct.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/datastruct.asm" -%include "include/constants.asm" +%include "include/datastruct.inc" +%include "include/constants.inc" -%ifndef MB_MGR_DATASTRUCT_ASM_INCLUDED -%define MB_MGR_DATASTRUCT_ASM_INCLUDED +%ifndef MB_MGR_DATASTRUCT_INC_INCLUDED +%define MB_MGR_DATASTRUCT_INC_INCLUDED ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Define constants @@ -493,4 +493,4 @@ _snow3g_args_FSM_3 equ _snow3g_args + __snow3g_arg_FSM_3 _snow3g_args_LD_ST_MASK equ _snow3g_args + __snow3g_arg_LD_ST_MASK _snow3g_args_byte_length equ _snow3g_args + __snow3g_arg_byte_length -%endif ;; MB_MGR_DATASTRUCT_ASM_INCLUDED +%endif ;; MB_MGR_DATASTRUCT_INC_INCLUDED diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index bc480ce889d93a955efe6c6755619ec7538f9e5d..b942a920d97708afe5ee973326426e1c54307f5f 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -41,16 +41,23 @@ #include +#ifndef __aarch64__ #include "include/clear_regs_mem.h" #include "include/des.h" +#else /* __aarch64__ */ +#include "aarch64/clear_regs_mem_aarch64.h" +#endif /* __aarch64__ */ #include "ipsec-mb.h" #include "include/error.h" #include "include/snow3g_submit.h" +#ifndef __aarch64__ #include "include/job_api_gcm.h" #include "include/job_api_snowv.h" #include "include/job_api_kasumi.h" +#endif /* __aarch64__ */ #include "include/mb_mgr_job_check.h" /* is_job_invalid() */ +#ifndef __aarch64__ #define CRC(func, state, job) *((uint32_t *)job->auth_tag_output) = \ func(state, job->src + job->hash_start_src_offset_in_bytes, \ job->msg_len_to_hash_in_bytes) @@ -181,99 +188,35 @@ __forceinline IMB_JOB * SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) /* ========================================================================= */ /* AES-GCM */ /* ========================================================================= */ -__forceinline IMB_JOB *SUBMIT_JOB_AES_GCM_DEC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_GCM_DEC(IMB_MGR *state, IMB_JOB *job, + const uint64_t key_sz) { - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; + if (16 == key_sz) + return AES_GCM_DEC_IV_128(state, job); + else if (24 == key_sz) + return AES_GCM_DEC_IV_192(state, job); + else + return AES_GCM_DEC_IV_256(state, job); } -__forceinline IMB_JOB *SUBMIT_JOB_AES_GCM_ENC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_GCM_ENC(IMB_MGR *state, IMB_JOB *job, + const uint64_t key_sz) { - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; + if (16 == key_sz) + return AES_GCM_ENC_IV_128(state, job); + else if (24 == key_sz) + return AES_GCM_ENC_IV_192(state, job); + else + return AES_GCM_ENC_IV_256(state, job); } + /* ========================================================================= */ /* AES-CTR */ /* ========================================================================= */ -__forceinline IMB_JOB *SUBMIT_JOB_AES_CTR(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_CTR(IMB_JOB *job, + const uint64_t key_sz) { - if (IMB_KEY_128_BYTES == job->key_len_in_bytes) { + if (IMB_KEY_128_BYTES == key_sz) { #ifdef SUBMIT_JOB_AES_CTR_128 SUBMIT_JOB_AES_CTR_128(job); #else @@ -284,7 +227,7 @@ __forceinline IMB_JOB *SUBMIT_JOB_AES_CTR(IMB_JOB *job) job->msg_len_to_cipher_in_bytes, job->iv_len_in_bytes); #endif - } else if (IMB_KEY_192_BYTES == job->key_len_in_bytes) { + } else if (IMB_KEY_192_BYTES == key_sz) { #ifdef SUBMIT_JOB_AES_CTR_192 SUBMIT_JOB_AES_CTR_192(job); #else @@ -312,9 +255,10 @@ __forceinline IMB_JOB *SUBMIT_JOB_AES_CTR(IMB_JOB *job) return job; } -__forceinline IMB_JOB *SUBMIT_JOB_AES_CTR_BIT(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_CTR_BIT(IMB_JOB *job, + const uint64_t key_sz) { - if (IMB_KEY_128_BYTES == job->key_len_in_bytes) { + if (IMB_KEY_128_BYTES == key_sz) { #ifdef SUBMIT_JOB_AES_CTR_128_BIT SUBMIT_JOB_AES_CTR_128_BIT(job); #else @@ -326,7 +270,7 @@ __forceinline IMB_JOB *SUBMIT_JOB_AES_CTR_BIT(IMB_JOB *job) job->msg_len_to_cipher_in_bits, job->iv_len_in_bytes); #endif - } else if (IMB_KEY_192_BYTES == job->key_len_in_bytes) { + } else if (IMB_KEY_192_BYTES == key_sz) { #ifdef SUBMIT_JOB_AES_CTR_192_BIT SUBMIT_JOB_AES_CTR_192_BIT(job); #else @@ -401,22 +345,26 @@ __forceinline IMB_JOB *FLUSH_JOB_CUSTOM_HASH(IMB_JOB *job) { return JOB_CUSTOM_HASH(job); } +#endif /* __aarch64__ */ /* ========================================================================= */ /* Cipher submit & flush functions */ /* ========================================================================= */ -__forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job, + const IMB_CIPHER_MODE cipher_mode, + const uint64_t key_sz) { - if (IMB_CIPHER_GCM == job->cipher_mode) { - return SUBMIT_JOB_AES_GCM_ENC(state, job); - } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { - return submit_gcm_sgl_enc(state, job); - } else if (IMB_CIPHER_CBC == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { +#ifndef __aarch64__ + if (IMB_CIPHER_GCM == cipher_mode) { + return SUBMIT_JOB_AES_GCM_ENC(state, job, key_sz); + } else if (IMB_CIPHER_GCM_SGL == cipher_mode) { + return submit_gcm_sgl_enc(state, job, key_sz); + } else if (IMB_CIPHER_CBC == cipher_mode) { + if (16 == key_sz) { MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; return SUBMIT_JOB_AES_CBC_128_ENC(aes128_ooo, job); - } else if (24 == job->key_len_in_bytes) { + } else if (24 == key_sz) { MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; return SUBMIT_JOB_AES_CBC_192_ENC(aes192_ooo, job); @@ -425,28 +373,28 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_AES_CBC_256_ENC(aes256_ooo, job); } - } else if (IMB_CIPHER_CNTR == job->cipher_mode) { - return SUBMIT_JOB_AES_CTR(job); - } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_AES_CTR_BIT(job); - } else if (IMB_CIPHER_ECB == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + } else if (IMB_CIPHER_CNTR == cipher_mode) { + return SUBMIT_JOB_AES_CTR(job, key_sz); + } else if (IMB_CIPHER_CNTR_BITLEN == cipher_mode) { + return SUBMIT_JOB_AES_CTR_BIT(job, key_sz); + } else if (IMB_CIPHER_ECB == cipher_mode) { + if (16 == key_sz) { return SUBMIT_JOB_AES_ECB_128_ENC(job); - } else if (24 == job->key_len_in_bytes) { + } else if (24 == key_sz) { return SUBMIT_JOB_AES_ECB_192_ENC(job); } else { /* assume 32 */ return SUBMIT_JOB_AES_ECB_256_ENC(job); } - } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { - return submit_docsis_enc_job(state, job); - } else if (IMB_CIPHER_PON_AES_CNTR == job->cipher_mode) { + } else if (IMB_CIPHER_DOCSIS_SEC_BPI == cipher_mode) { + return submit_docsis_enc_job(state, job, key_sz); + } else if (IMB_CIPHER_PON_AES_CNTR == cipher_mode) { if (job->msg_len_to_cipher_in_bytes == 0) return SUBMIT_JOB_PON_ENC_NO_CTR(job); else return SUBMIT_JOB_PON_ENC(job); - } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { + } else if (IMB_CIPHER_CUSTOM == cipher_mode) { return SUBMIT_JOB_CUSTOM_CIPHER(job); - } else if (IMB_CIPHER_DES == job->cipher_mode) { + } else if (IMB_CIPHER_DES == cipher_mode) { #ifdef SUBMIT_JOB_DES_CBC_ENC MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; @@ -454,13 +402,13 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) #else return DES_CBC_ENC(job); #endif /* SUBMIT_JOB_DES_CBC_ENC */ - } else if (IMB_CIPHER_CHACHA20 == job->cipher_mode) { + } else if (IMB_CIPHER_CHACHA20 == cipher_mode) { return SUBMIT_JOB_CHACHA20_ENC_DEC(job); - } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { + } else if (IMB_CIPHER_CHACHA20_POLY1305 == cipher_mode) { return SUBMIT_JOB_CHACHA20_POLY1305(state, job); - } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == job->cipher_mode) { + } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == cipher_mode) { return SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); - } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { + } else if (IMB_CIPHER_DOCSIS_DES == cipher_mode) { #ifdef SUBMIT_JOB_DOCSIS_DES_ENC MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; @@ -469,7 +417,7 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) #else return DOCSIS_DES_ENC(job); #endif /* SUBMIT_JOB_DOCSIS_DES_ENC */ - } else if (IMB_CIPHER_DES3 == job->cipher_mode) { + } else if (IMB_CIPHER_DES3 == cipher_mode) { #ifdef SUBMIT_JOB_3DES_CBC_ENC MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; @@ -477,14 +425,17 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) #else return DES3_CBC_ENC(job); #endif - } else if (IMB_CIPHER_CCM == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + } else if (IMB_CIPHER_CCM == cipher_mode) { + if (16 == key_sz) { return AES_CNTR_CCM_128(job); } else { /* assume 32 */ return AES_CNTR_CCM_256(job); } - } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + } else if (IMB_CIPHER_ZUC_EEA3 == cipher_mode) { +#else /* __aarch64__ */ + if (IMB_CIPHER_ZUC_EEA3 == cipher_mode) { +#endif /* __aarch64__ */ + if (16 == key_sz) { MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); @@ -494,36 +445,41 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); } - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { + } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == cipher_mode) { #ifdef SUBMIT_JOB_SNOW3G_UEA2 return SUBMIT_JOB_SNOW3G_UEA2(state, job); #else return def_submit_snow3g_uea2_job(state, job); #endif - } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == job->cipher_mode) { +#ifndef __aarch64__ + } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == cipher_mode) { return submit_kasumi_uea1_job(state, job); - } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { + } else if (IMB_CIPHER_CBCS_1_9 == cipher_mode) { MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; return SUBMIT_JOB_AES128_CBCS_1_9_ENC(aes128_cbcs_ooo, job); - } else if (IMB_CIPHER_SNOW_V == job->cipher_mode) { + } else if (IMB_CIPHER_SNOW_V == cipher_mode) { return SUBMIT_JOB_SNOW_V(job); - } else if (IMB_CIPHER_SNOW_V_AEAD == job->cipher_mode) { + } else if (IMB_CIPHER_SNOW_V_AEAD == cipher_mode) { return submit_snow_v_aead_job(state, job); +#endif /* __aarch64__ */ } else { /* assume IMB_CIPHER_NULL */ job->status |= IMB_STATUS_COMPLETED_CIPHER; return job; } } -__forceinline IMB_JOB *FLUSH_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *FLUSH_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job, + const IMB_CIPHER_MODE cipher_mode, + const uint64_t key_sz) { - if (IMB_CIPHER_CBC == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { +#ifndef __aarch64__ + if (IMB_CIPHER_CBC == cipher_mode) { + if (16 == key_sz) { MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; return FLUSH_JOB_AES_CBC_128_ENC(aes128_ooo); - } else if (24 == job->key_len_in_bytes) { + } else if (24 == key_sz) { MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; return FLUSH_JOB_AES_CBC_192_ENC(aes192_ooo); @@ -532,30 +488,33 @@ __forceinline IMB_JOB *FLUSH_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_AES_CBC_256_ENC(aes256_ooo); } - } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { - return flush_docsis_enc_job(state, job); + } else if (IMB_CIPHER_DOCSIS_SEC_BPI == cipher_mode) { + return flush_docsis_enc_job(state, job, key_sz); #ifdef FLUSH_JOB_DES_CBC_ENC - } else if (IMB_CIPHER_DES == job->cipher_mode) { + } else if (IMB_CIPHER_DES == cipher_mode) { MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; return FLUSH_JOB_DES_CBC_ENC(des_enc_ooo); #endif /* FLUSH_JOB_DES_CBC_ENC */ #ifdef FLUSH_JOB_3DES_CBC_ENC - } else if (IMB_CIPHER_DES3 == job->cipher_mode) { + } else if (IMB_CIPHER_DES3 == cipher_mode) { MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; return FLUSH_JOB_3DES_CBC_ENC(des3_enc_ooo); #endif /* FLUSH_JOB_3DES_CBC_ENC */ #ifdef FLUSH_JOB_DOCSIS_DES_ENC - } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { + } else if (IMB_CIPHER_DOCSIS_DES == cipher_mode) { MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; return FLUSH_JOB_DOCSIS_DES_ENC(docsis_des_enc_ooo); #endif /* FLUSH_JOB_DOCSIS_DES_ENC */ - } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { + } else if (IMB_CIPHER_CUSTOM == cipher_mode) { return FLUSH_JOB_CUSTOM_CIPHER(job); - } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + } else if (IMB_CIPHER_ZUC_EEA3 == cipher_mode) { +#else /* __aarch64__ */ + if (IMB_CIPHER_ZUC_EEA3 == cipher_mode) { +#endif /* __aarch64__ */ + if (16 == key_sz) { MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); @@ -565,12 +524,14 @@ __forceinline IMB_JOB *FLUSH_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_ZUC256_EEA3(zuc256_eea3_ooo); } - } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { +#ifndef __aarch64__ + } else if (IMB_CIPHER_CBCS_1_9 == cipher_mode) { MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; return FLUSH_JOB_AES128_CBCS_1_9_ENC(aes128_cbcs_ooo); +#endif /* __aarch64__ */ #ifdef FLUSH_JOB_SNOW3G_UEA2 - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { + } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == cipher_mode) { return FLUSH_JOB_SNOW3G_UEA2(state); #endif /** @@ -582,40 +543,43 @@ __forceinline IMB_JOB *FLUSH_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) } } -__forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job, + const IMB_CIPHER_MODE cipher_mode, + const uint64_t key_sz) { - if (IMB_CIPHER_GCM == job->cipher_mode) { - return SUBMIT_JOB_AES_GCM_DEC(state, job); - } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { - return submit_gcm_sgl_dec(state, job); - } else if (IMB_CIPHER_CBC == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { +#ifndef __aarch64__ + if (IMB_CIPHER_GCM == cipher_mode) { + return SUBMIT_JOB_AES_GCM_DEC(state, job, key_sz); + } else if (IMB_CIPHER_GCM_SGL == cipher_mode) { + return submit_gcm_sgl_dec(state, job, key_sz); + } else if (IMB_CIPHER_CBC == cipher_mode) { + if (16 == key_sz) { return SUBMIT_JOB_AES_CBC_128_DEC(job); - } else if (24 == job->key_len_in_bytes) { + } else if (24 == key_sz) { return SUBMIT_JOB_AES_CBC_192_DEC(job); } else { /* assume 32 */ return SUBMIT_JOB_AES_CBC_256_DEC(job); } - } else if (IMB_CIPHER_CNTR == job->cipher_mode) { - return SUBMIT_JOB_AES_CTR(job); - } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_AES_CTR_BIT(job); - } else if (IMB_CIPHER_ECB == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + } else if (IMB_CIPHER_CNTR == cipher_mode) { + return SUBMIT_JOB_AES_CTR(job, key_sz); + } else if (IMB_CIPHER_CNTR_BITLEN == cipher_mode) { + return SUBMIT_JOB_AES_CTR_BIT(job, key_sz); + } else if (IMB_CIPHER_ECB == cipher_mode) { + if (16 == key_sz) { return SUBMIT_JOB_AES_ECB_128_DEC(job); - } else if (24 == job->key_len_in_bytes) { + } else if (24 == key_sz) { return SUBMIT_JOB_AES_ECB_192_DEC(job); } else { /* assume 32 */ return SUBMIT_JOB_AES_ECB_256_DEC(job); } - } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { - return submit_docsis_dec_job(state, job); - } else if (IMB_CIPHER_PON_AES_CNTR == job->cipher_mode) { + } else if (IMB_CIPHER_DOCSIS_SEC_BPI == cipher_mode) { + return submit_docsis_dec_job(state, job, key_sz); + } else if (IMB_CIPHER_PON_AES_CNTR == cipher_mode) { if (job->msg_len_to_cipher_in_bytes == 0) return SUBMIT_JOB_PON_DEC_NO_CTR(job); else return SUBMIT_JOB_PON_DEC(job); - } else if (IMB_CIPHER_DES == job->cipher_mode) { + } else if (IMB_CIPHER_DES == cipher_mode) { #ifdef SUBMIT_JOB_DES_CBC_DEC MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; @@ -624,13 +588,13 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) (void) state; return DES_CBC_DEC(job); #endif /* SUBMIT_JOB_DES_CBC_DEC */ - } else if (IMB_CIPHER_CHACHA20 == job->cipher_mode) { + } else if (IMB_CIPHER_CHACHA20 == cipher_mode) { return SUBMIT_JOB_CHACHA20_ENC_DEC(job); - } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { + } else if (IMB_CIPHER_CHACHA20_POLY1305 == cipher_mode) { return SUBMIT_JOB_CHACHA20_POLY1305(state, job); - } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == job->cipher_mode) { + } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == cipher_mode) { return SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); - } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { + } else if (IMB_CIPHER_DOCSIS_DES == cipher_mode) { #ifdef SUBMIT_JOB_DOCSIS_DES_DEC MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; @@ -639,7 +603,7 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) #else return DOCSIS_DES_DEC(job); #endif /* SUBMIT_JOB_DOCSIS_DES_DEC */ - } else if (IMB_CIPHER_DES3 == job->cipher_mode) { + } else if (IMB_CIPHER_DES3 == cipher_mode) { #ifdef SUBMIT_JOB_3DES_CBC_DEC MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; @@ -647,16 +611,19 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) #else return DES3_CBC_DEC(job); #endif - } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { + } else if (IMB_CIPHER_CUSTOM == cipher_mode) { return SUBMIT_JOB_CUSTOM_CIPHER(job); - } else if (IMB_CIPHER_CCM == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + } else if (IMB_CIPHER_CCM == cipher_mode) { + if (16 == key_sz) { return AES_CNTR_CCM_128(job); } else { /* assume 32 */ return AES_CNTR_CCM_256(job); } - } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + } else if (IMB_CIPHER_ZUC_EEA3 == cipher_mode) { +#else /* __aarch64__ */ + if (IMB_CIPHER_ZUC_EEA3 == cipher_mode) { +#endif /* __aarch64__ */ + if (16 == key_sz) { MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); @@ -666,20 +633,22 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); } - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { + } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == cipher_mode) { #ifdef SUBMIT_JOB_SNOW3G_UEA2 return SUBMIT_JOB_SNOW3G_UEA2(state, job); #else return def_submit_snow3g_uea2_job(state, job); #endif - } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == job->cipher_mode) { +#ifndef __aarch64__ + } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == cipher_mode) { return submit_kasumi_uea1_job(state, job); - } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { + } else if (IMB_CIPHER_CBCS_1_9 == cipher_mode) { return SUBMIT_JOB_AES128_CBCS_1_9_DEC(job); - } else if (IMB_CIPHER_SNOW_V == job->cipher_mode) { + } else if (IMB_CIPHER_SNOW_V == cipher_mode) { return SUBMIT_JOB_SNOW_V(job); - } else if (IMB_CIPHER_SNOW_V_AEAD == job->cipher_mode) { + } else if (IMB_CIPHER_SNOW_V_AEAD == cipher_mode) { return submit_snow_v_aead_job(state, job); +#endif /* __aarch64__ */ } else { /* assume IMB_CIPHER_NULL */ job->status |= IMB_STATUS_COMPLETED_CIPHER; @@ -687,15 +656,20 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) } } -__forceinline IMB_JOB *FLUSH_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *FLUSH_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job, + const IMB_CIPHER_MODE cipher_mode, + const uint64_t key_sz) { + (void) job; + #ifdef FLUSH_JOB_SNOW3G_UEA2 - if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) + if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == cipher_mode) return FLUSH_JOB_SNOW3G_UEA2(state); #endif +#ifndef __aarch64__ #ifdef FLUSH_JOB_DES_CBC_DEC - if (IMB_CIPHER_DES == job->cipher_mode) { + if (IMB_CIPHER_DES == cipher_mode) { MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; return FLUSH_JOB_DES_CBC_DEC(des_dec_ooo); @@ -703,7 +677,7 @@ __forceinline IMB_JOB *FLUSH_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) #endif /* FLUSH_JOB_DES_CBC_DEC */ #ifdef FLUSH_JOB_3DES_CBC_DEC - if (IMB_CIPHER_DES3 == job->cipher_mode) { + if (IMB_CIPHER_DES3 == cipher_mode) { MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; return FLUSH_JOB_3DES_CBC_DEC(des3_dec_ooo); @@ -712,15 +686,16 @@ __forceinline IMB_JOB *FLUSH_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) #ifdef FLUSH_JOB_DOCSIS_DES_DEC - if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { + if (IMB_CIPHER_DOCSIS_DES == cipher_mode) { MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; return FLUSH_JOB_DOCSIS_DES_DEC(docsis_des_dec_ooo); } #endif /* FLUSH_JOB_DOCSIS_DES_DEC */ +#endif /* __aarch64__ */ - if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { + if (IMB_CIPHER_ZUC_EEA3 == cipher_mode) { + if (16 == key_sz) { MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); @@ -736,258 +711,2039 @@ __forceinline IMB_JOB *FLUSH_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) } /* ========================================================================= */ -/* Hash submit & flush functions */ +/* Generate specialized submit cipher functions and create a table */ /* ========================================================================= */ -__forceinline -IMB_JOB * -SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) +/* ========================= */ +/* ======== DECRYPT ======== */ +/* ========================= */ + +#ifndef __aarch64__ +/* AES-CBC */ +static IMB_JOB *submit_cipher_dec_aes_cbc_128(IMB_MGR *state, IMB_JOB *job) { - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; - MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; - MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; - MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) - MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; -#endif + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBC, + IMB_KEY_128_BYTES); +} +static IMB_JOB *submit_cipher_dec_aes_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBC, + IMB_KEY_192_BYTES); +} - switch (job->hash_alg) { - case IMB_AUTH_HMAC_SHA_1: - return SUBMIT_JOB_HMAC(hmac_sha_1_ooo, job); - case IMB_AUTH_HMAC_SHA_224: - return SUBMIT_JOB_HMAC_SHA_224(hmac_sha_224_ooo, job); - case IMB_AUTH_HMAC_SHA_256: - return SUBMIT_JOB_HMAC_SHA_256(hmac_sha_256_ooo, job); - case IMB_AUTH_HMAC_SHA_384: - return SUBMIT_JOB_HMAC_SHA_384(hmac_sha_384_ooo, job); - case IMB_AUTH_HMAC_SHA_512: - return SUBMIT_JOB_HMAC_SHA_512(hmac_sha_512_ooo, job); - case IMB_AUTH_AES_XCBC: - return SUBMIT_JOB_AES_XCBC(aes_xcbc_ooo, job); - case IMB_AUTH_MD5: - return SUBMIT_JOB_HMAC_MD5(hmac_md5_ooo, job); - case IMB_AUTH_CUSTOM: - return SUBMIT_JOB_CUSTOM_HASH(job); - case IMB_AUTH_AES_CCM: - if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES128_CCM_AUTH(aes_ccm_ooo, job); - } else { /* assume 32 */ - return SUBMIT_JOB_AES256_CCM_AUTH(aes256_ccm_ooo, job); - } - case IMB_AUTH_AES_CMAC: - /* - * CMAC OOO MGR assumes job len in bits - * (for CMAC length is provided in bytes) - */ - job->msg_len_to_hash_in_bits = - job->msg_len_to_hash_in_bytes * 8; - return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); - case IMB_AUTH_AES_CMAC_BITLEN: - return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); - case IMB_AUTH_AES_CMAC_256: - job->msg_len_to_hash_in_bits = - job->msg_len_to_hash_in_bytes * 8; - return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); - case IMB_AUTH_SHA_1: - return SUBMIT_JOB_SHA1(sha_1_ooo, job); - case IMB_AUTH_SHA_224: - return SUBMIT_JOB_SHA224(sha_224_ooo, job); - case IMB_AUTH_SHA_256: - return SUBMIT_JOB_SHA256(sha_256_ooo, job); - case IMB_AUTH_SHA_384: - return SUBMIT_JOB_SHA384(sha_384_ooo, job); - case IMB_AUTH_SHA_512: - return SUBMIT_JOB_SHA512(sha_512_ooo, job); - case IMB_AUTH_ZUC_EIA3_BITLEN: - return SUBMIT_JOB_ZUC_EIA3(zuc_eia3_ooo, job); - case IMB_AUTH_ZUC256_EIA3_BITLEN: - return SUBMIT_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job, - job->auth_tag_output_len_in_bytes); - case IMB_AUTH_SNOW3G_UIA2_BITLEN: -#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) - return SUBMIT_JOB_SNOW3G_UIA2(snow3g_uia2_ooo, job); -#else - IMB_SNOW3G_F9_1_BUFFER(state, (const snow3g_key_schedule_t *) - job->u.SNOW3G_UIA2._key, - job->u.SNOW3G_UIA2._iv, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bits, - job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -#endif - case IMB_AUTH_KASUMI_UIA1: - IMB_KASUMI_F9_1_BUFFER(state, (const kasumi_key_sched_t *) - job->u.KASUMI_UIA1._key, - job->src + job->hash_start_src_offset_in_bytes, - (const uint32_t) job->msg_len_to_hash_in_bytes, - job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_AES_GMAC_128: - process_gmac(state, job, IMB_KEY_128_BYTES); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_AES_GMAC_192: - process_gmac(state, job, IMB_KEY_192_BYTES); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_AES_GMAC_256: - process_gmac(state, job, IMB_KEY_256_BYTES); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_POLY1305: - POLY1305_MAC(job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC32_ETHERNET_FCS: - CRC(IMB_CRC32_ETHERNET_FCS, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC32_SCTP: - CRC(IMB_CRC32_SCTP, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: - CRC(IMB_CRC32_WIMAX_OFDMA_DATA, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC24_LTE_A: - CRC(IMB_CRC24_LTE_A, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC24_LTE_B: - CRC(IMB_CRC24_LTE_B, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC16_X25: - CRC(IMB_CRC16_X25, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC16_FP_DATA: - CRC(IMB_CRC16_FP_DATA, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC11_FP_HEADER: - CRC(IMB_CRC11_FP_HEADER, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC10_IUUP_DATA: - CRC(IMB_CRC10_IUUP_DATA, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: - CRC(IMB_CRC8_WIMAX_OFDMA_HCS, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC7_FP_HEADER: - CRC(IMB_CRC7_FP_HEADER, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC6_IUUP_HEADER: - CRC(IMB_CRC6_IUUP_HEADER, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_GHASH: - return process_ghash(state, job); - default: - /** - * assume IMB_AUTH_GCM, IMB_AUTH_PON_CRC_BIP, - * IMB_AUTH_SNOW_V_AEAD or IMB_AUTH_NULL - */ - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - } +static IMB_JOB *submit_cipher_dec_aes_cbc_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBC, + IMB_KEY_256_BYTES); } -__forceinline -IMB_JOB * -FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) +/* AES-CTR */ +static IMB_JOB *submit_cipher_dec_aes_ctr_128(IMB_MGR *state, IMB_JOB *job) { - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; - MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; - MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; - MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) - MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; -#endif + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_128_BYTES); +} - switch (job->hash_alg) { - case IMB_AUTH_HMAC_SHA_1: - return FLUSH_JOB_HMAC(hmac_sha_1_ooo); - case IMB_AUTH_HMAC_SHA_224: - return FLUSH_JOB_HMAC_SHA_224(hmac_sha_224_ooo); - case IMB_AUTH_HMAC_SHA_256: - return FLUSH_JOB_HMAC_SHA_256(hmac_sha_256_ooo); - case IMB_AUTH_HMAC_SHA_384: - return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); - case IMB_AUTH_HMAC_SHA_512: - return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); - case IMB_AUTH_SHA_1: - return FLUSH_JOB_SHA1(sha_1_ooo, job); - case IMB_AUTH_SHA_224: - return FLUSH_JOB_SHA224(sha_224_ooo, job); - case IMB_AUTH_SHA_256: - return FLUSH_JOB_SHA256(sha_256_ooo, job); - case IMB_AUTH_SHA_384: - return FLUSH_JOB_SHA384(sha_384_ooo, job); - case IMB_AUTH_SHA_512: - return FLUSH_JOB_SHA512(sha_512_ooo, job); - case IMB_AUTH_AES_XCBC: - return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); - case IMB_AUTH_MD5: - return FLUSH_JOB_HMAC_MD5(hmac_md5_ooo); - case IMB_AUTH_CUSTOM: - return FLUSH_JOB_CUSTOM_HASH(job); - case IMB_AUTH_AES_CCM: - if (16 == job->key_len_in_bytes) { - return FLUSH_JOB_AES128_CCM_AUTH(aes_ccm_ooo); - } else { /* assume 32 */ - return FLUSH_JOB_AES256_CCM_AUTH(aes256_ccm_ooo); - } +static IMB_JOB *submit_cipher_dec_aes_ctr_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_ctr_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* NULL */ +static IMB_JOB *submit_cipher_dec_null(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_NULL, + IMB_KEY_128_BYTES); +} + + #ifndef __aarch64__ +/* AES DOCSIS */ +static IMB_JOB *submit_cipher_dec_aes_docsis_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_docsis_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_256_BYTES); +} + +/* AES-GCM */ +#define submit_cipher_dec_aes_gcm_128 AES_GCM_DEC_IV_128 +#define submit_cipher_dec_aes_gcm_192 AES_GCM_DEC_IV_192 +#define submit_cipher_dec_aes_gcm_256 AES_GCM_DEC_IV_256 + +/* CUSTOM */ +static IMB_JOB *submit_cipher_dec_custom(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CUSTOM, + IMB_KEY_128_BYTES); +} + +/* DES */ +static IMB_JOB *submit_cipher_dec_des_cbc_64(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DES, + IMB_KEY_64_BYTES); +} + +/* DES DOCSIS */ +static IMB_JOB *submit_cipher_dec_des_docsis_64(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DOCSIS_DES, + IMB_KEY_64_BYTES); +} + +/* AES-CCM */ +static IMB_JOB *submit_cipher_dec_aes_ccm_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CCM, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_ccm_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CCM, + IMB_KEY_256_BYTES); +} + +/* 3DES */ +static IMB_JOB *submit_cipher_dec_des3_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DES3, + IMB_KEY_192_BYTES); +} + +/* PON AES-CTR */ +static IMB_JOB *submit_cipher_dec_aes_ctr_pon_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_PON_AES_CNTR, + IMB_KEY_128_BYTES); +} + +/* AES-ECB */ +static IMB_JOB *submit_cipher_dec_aes_ecb_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ECB, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_ecb_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ECB, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_ecb_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ECB, + IMB_KEY_256_BYTES); +} + +/* AES-CTR BITS */ +static IMB_JOB *submit_cipher_dec_aes_ctr_128_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_ctr_192_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_ctr_256_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* ZUC EEA3 */ +static IMB_JOB *submit_cipher_dec_zuc_eea3_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_dec_zuc_eea3_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_256_BYTES); +} + +/* SNOW3G UEA2 */ +static IMB_JOB *submit_cipher_dec_snow3g_uea2_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_SNOW3G_UEA2_BITLEN, + IMB_KEY_128_BYTES); +} + +#ifndef __aarch64__ +/* KASUMI F8 UEA1 */ +static IMB_JOB *submit_cipher_dec_kasumi_uea1_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_KASUMI_UEA1_BITLEN, + IMB_KEY_128_BYTES); +} + +/* AES-CBCS-1-9 */ +static IMB_JOB *submit_cipher_dec_aes_cbcs_1_9(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBCS_1_9, + IMB_KEY_128_BYTES); +} + +/* CHACHA20 */ +static IMB_JOB *submit_cipher_dec_chacha20(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CHACHA20, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 */ +static IMB_JOB * +submit_cipher_dec_chacha20_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CHACHA20_POLY1305, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 SGL */ +static IMB_JOB * +submit_cipher_dec_chacha20_poly1305_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, + IMB_CIPHER_CHACHA20_POLY1305_SGL, + IMB_KEY_256_BYTES); +} + +/* SNOW-V */ +static IMB_JOB *submit_cipher_dec_snow_v(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_SNOW_V, + IMB_KEY_256_BYTES); +} + +/* SNOW-V AEAD */ +static IMB_JOB *submit_cipher_dec_snow_v_aead(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_SNOW_V_AEAD, + IMB_KEY_256_BYTES); +} + +/* AES-GCM SGL */ +static IMB_JOB *submit_cipher_dec_aes_gcm_128_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_gcm_192_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_dec_aes_gcm_256_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* ========================= */ +/* ======== ENCRYPT ======== */ +/* ========================= */ + +#ifndef __aarch64__ +/* AES-CBC */ +static IMB_JOB *submit_cipher_enc_aes_cbc_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBC, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBC, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_cbc_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBC, + IMB_KEY_256_BYTES); +} + +/* AES-CTR */ +static IMB_JOB *submit_cipher_enc_aes_ctr_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_ctr_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_ctr_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* NULL */ +static IMB_JOB *submit_cipher_enc_null(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_NULL, + IMB_KEY_128_BYTES); +} + +#ifndef __aarch64__ +/* AES DOCSIS */ +static IMB_JOB *submit_cipher_enc_aes_docsis_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_docsis_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_256_BYTES); +} + +/* AES-GCM */ +#define submit_cipher_enc_aes_gcm_128 AES_GCM_ENC_IV_128 +#define submit_cipher_enc_aes_gcm_192 AES_GCM_ENC_IV_192 +#define submit_cipher_enc_aes_gcm_256 AES_GCM_ENC_IV_256 + +/* CUSTOM */ +static IMB_JOB *submit_cipher_enc_custom(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CUSTOM, + IMB_KEY_128_BYTES); +} + +/* DES */ +static IMB_JOB *submit_cipher_enc_des_cbc_64(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DES, + IMB_KEY_64_BYTES); +} + +/* DES DOCSIS */ +static IMB_JOB *submit_cipher_enc_des_docsis_64(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DOCSIS_DES, + IMB_KEY_64_BYTES); +} + +/* AES-CCM */ +static IMB_JOB *submit_cipher_enc_aes_ccm_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CCM, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_ccm_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CCM, + IMB_KEY_256_BYTES); +} + +/* 3DES */ +static IMB_JOB *submit_cipher_enc_des3_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DES3, + IMB_KEY_192_BYTES); +} + +/* PON AES-CTR */ +static IMB_JOB *submit_cipher_enc_aes_ctr_pon_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_PON_AES_CNTR, + IMB_KEY_128_BYTES); +} + +/* AES-ECB */ +static IMB_JOB *submit_cipher_enc_aes_ecb_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ECB, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_ecb_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ECB, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_ecb_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ECB, + IMB_KEY_256_BYTES); +} + +/* AES-CTR BITS */ +static IMB_JOB *submit_cipher_enc_aes_ctr_128_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_ctr_192_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_ctr_256_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* ZUC EEA3 */ +static IMB_JOB *submit_cipher_enc_zuc_eea3_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_zuc_eea3_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_256_BYTES); +} + +/* SNOW3G UEA2 */ +static IMB_JOB *submit_cipher_enc_snow3g_uea2_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_SNOW3G_UEA2_BITLEN, + IMB_KEY_128_BYTES); +} + +#ifndef __aarch64__ +/* KASUMI F8 UEA1 */ +static IMB_JOB *submit_cipher_enc_kasumi_uea1_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_KASUMI_UEA1_BITLEN, + IMB_KEY_128_BYTES); +} + +/* AES-CBCS-1-9 */ +static IMB_JOB *submit_cipher_enc_aes_cbcs_1_9(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBCS_1_9, + IMB_KEY_128_BYTES); +} + +/* CHACHA20 */ +static IMB_JOB *submit_cipher_enc_chacha20(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CHACHA20, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 */ +static IMB_JOB * +submit_cipher_enc_chacha20_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CHACHA20_POLY1305, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 SGL */ +static IMB_JOB * +submit_cipher_enc_chacha20_poly1305_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, + IMB_CIPHER_CHACHA20_POLY1305_SGL, + IMB_KEY_256_BYTES); +} + +/* SNOW-V */ +static IMB_JOB *submit_cipher_enc_snow_v(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_SNOW_V, + IMB_KEY_256_BYTES); +} + +/* SNOW-V AEAD */ +static IMB_JOB *submit_cipher_enc_snow_v_aead(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_SNOW_V_AEAD, + IMB_KEY_256_BYTES); +} + +/* AES-GCM SGL */ +static IMB_JOB *submit_cipher_enc_aes_gcm_128_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_gcm_192_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *submit_cipher_enc_aes_gcm_256_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* + * Four entries per algorithm (different key sizes), + * algorithms in the same order IMB_CIPHER_MODE + * index 0 - key size from 0 to 64-bits + * index 1 - key size from 65 to 128-bits + * index 2 - key size from 129 to 192-bits + * index 3 - key size from 193 to 256-bits + */ +typedef IMB_JOB *(*submit_flush_fn_t)(IMB_MGR *, IMB_JOB *); + +#define ENCRYPT_DECRYPT_GAP 32 + +static const submit_flush_fn_t tab_submit_cipher[] = { + /* ========================= */ + /* === DECRYPT DIRECTION === */ + /* ========================= */ + + /* [0] keep empty - enums start from value 1 */ + NULL, NULL, NULL, NULL, +#ifndef __aarch64__ + /* [1] AES-CBC */ + submit_cipher_dec_null, + submit_cipher_dec_aes_cbc_128, + submit_cipher_dec_aes_cbc_192, + submit_cipher_dec_aes_cbc_256, + /* [2] AES-CBC */ + submit_cipher_dec_null, + submit_cipher_dec_aes_ctr_128, + submit_cipher_dec_aes_ctr_192, + submit_cipher_dec_aes_ctr_256, +#else /* __aarch64__ */ + /* [1] - [2] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [3] NULL */ + submit_cipher_dec_null, + submit_cipher_dec_null, + submit_cipher_dec_null, + submit_cipher_dec_null, +#ifndef __aarch64__ + /* [4] DOCSIS SEC BPI */ + submit_cipher_dec_null, + submit_cipher_dec_aes_docsis_128, + submit_cipher_dec_null, + submit_cipher_dec_aes_docsis_256, + /* [5] AES-GCM */ + submit_cipher_dec_null, + submit_cipher_dec_aes_gcm_128, + submit_cipher_dec_aes_gcm_192, + submit_cipher_dec_aes_gcm_256, + /* [6] CUSTOM */ + submit_cipher_dec_custom, + submit_cipher_dec_custom, + submit_cipher_dec_custom, + submit_cipher_dec_custom, + /* [7] DES */ + submit_cipher_dec_des_cbc_64, + submit_cipher_dec_des_cbc_64, + submit_cipher_dec_des_cbc_64, + submit_cipher_dec_des_cbc_64, + /* [8] DOCSIS DES */ + submit_cipher_dec_des_docsis_64, + submit_cipher_dec_des_docsis_64, + submit_cipher_dec_des_docsis_64, + submit_cipher_dec_des_docsis_64, + /* [9] AES-CCM */ + submit_cipher_dec_null, + submit_cipher_dec_aes_ccm_128, + submit_cipher_dec_null, + submit_cipher_dec_aes_ccm_256, + /* [10] 3DES */ + submit_cipher_dec_des3_cbc_192, + submit_cipher_dec_des3_cbc_192, + submit_cipher_dec_des3_cbc_192, + submit_cipher_dec_des3_cbc_192, + /* [11] PON AES-CTR */ + submit_cipher_dec_aes_ctr_pon_128, + submit_cipher_dec_aes_ctr_pon_128, + submit_cipher_dec_aes_ctr_pon_128, + submit_cipher_dec_aes_ctr_pon_128, + /* [12] AES-ECB */ + submit_cipher_dec_null, + submit_cipher_dec_aes_ecb_128, + submit_cipher_dec_aes_ecb_192, + submit_cipher_dec_aes_ecb_256, + /* [13] AES-CTR BITLEN */ + submit_cipher_dec_null, + submit_cipher_dec_aes_ctr_128_bit, + submit_cipher_dec_aes_ctr_192_bit, + submit_cipher_dec_aes_ctr_256_bit, +#else /* __aarch64__ */ + /* [4] - [13] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [14] ZUC EEA3 */ + submit_cipher_dec_null, + submit_cipher_dec_zuc_eea3_128, + submit_cipher_dec_null, + submit_cipher_dec_zuc_eea3_256, + /* [15] SNOW3G UEA2 */ + submit_cipher_dec_snow3g_uea2_bit, + submit_cipher_dec_snow3g_uea2_bit, + submit_cipher_dec_snow3g_uea2_bit, + submit_cipher_dec_snow3g_uea2_bit, +#ifndef __aarch64__ + /* [16] KASUMI F8 UEA1 */ + submit_cipher_dec_kasumi_uea1_bit, + submit_cipher_dec_kasumi_uea1_bit, + submit_cipher_dec_kasumi_uea1_bit, + submit_cipher_dec_kasumi_uea1_bit, + /* [17] AES-CBCS-1-9 */ + submit_cipher_dec_aes_cbcs_1_9, + submit_cipher_dec_aes_cbcs_1_9, + submit_cipher_dec_aes_cbcs_1_9, + submit_cipher_dec_aes_cbcs_1_9, + /* [18] CHACHA20 */ + submit_cipher_dec_chacha20, + submit_cipher_dec_chacha20, + submit_cipher_dec_chacha20, + submit_cipher_dec_chacha20, + /* [19] CHACHA20-POLY1305 */ + submit_cipher_dec_chacha20_poly1305, + submit_cipher_dec_chacha20_poly1305, + submit_cipher_dec_chacha20_poly1305, + submit_cipher_dec_chacha20_poly1305, + /* [20] CHACHA20-POLY1305 SGL */ + submit_cipher_dec_chacha20_poly1305_sgl, + submit_cipher_dec_chacha20_poly1305_sgl, + submit_cipher_dec_chacha20_poly1305_sgl, + submit_cipher_dec_chacha20_poly1305_sgl, + /* [21] SNOW-V */ + submit_cipher_dec_snow_v, + submit_cipher_dec_snow_v, + submit_cipher_dec_snow_v, + submit_cipher_dec_snow_v, + /* [22] SNOW-V AEAD */ + submit_cipher_dec_snow_v_aead, + submit_cipher_dec_snow_v_aead, + submit_cipher_dec_snow_v_aead, + submit_cipher_dec_snow_v_aead, + /* [23] AES-GCM SGL */ + submit_cipher_dec_null, + submit_cipher_dec_aes_gcm_128_sgl, + submit_cipher_dec_aes_gcm_192_sgl, + submit_cipher_dec_aes_gcm_256_sgl, +#else /* __aarch64__ */ + /* [16] - [23] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + + /* add new cipher decrypt here */ + + /* [24] NULL */ + NULL, NULL, NULL, NULL, + /* [25] NULL */ + NULL, NULL, NULL, NULL, + /* [26] NULL */ + NULL, NULL, NULL, NULL, + /* [27] NULL */ + NULL, NULL, NULL, NULL, + /* [28] NULL */ + NULL, NULL, NULL, NULL, + /* [29] NULL */ + NULL, NULL, NULL, NULL, + /* [30] NULL */ + NULL, NULL, NULL, NULL, + /* [31] NULL */ + NULL, NULL, NULL, NULL, + + /* ========================= */ + /* === ENCRYPT DIRECTION === */ + /* ========================= */ + + /* [0] keep empty - enums start from value 1 */ + NULL, NULL, NULL, NULL, +#ifndef __aarch64__ + /* [1] AES-CBC */ + submit_cipher_enc_null, + submit_cipher_enc_aes_cbc_128, + submit_cipher_enc_aes_cbc_192, + submit_cipher_enc_aes_cbc_256, + /* [2] AES-CBC */ + submit_cipher_enc_null, + submit_cipher_enc_aes_ctr_128, + submit_cipher_enc_aes_ctr_192, + submit_cipher_enc_aes_ctr_256, +#else /* __aarch64__ */ + /* [1] - [2] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [3] NULL */ + submit_cipher_enc_null, + submit_cipher_enc_null, + submit_cipher_enc_null, + submit_cipher_enc_null, +#ifndef __aarch64__ + /* [4] DOCSIS SEC BPI */ + submit_cipher_enc_null, + submit_cipher_enc_aes_docsis_128, + submit_cipher_enc_null, + submit_cipher_enc_aes_docsis_256, + /* [5] AES-GCM */ + submit_cipher_enc_null, + submit_cipher_enc_aes_gcm_128, + submit_cipher_enc_aes_gcm_192, + submit_cipher_enc_aes_gcm_256, + /* [6] CUSTOM */ + submit_cipher_enc_custom, + submit_cipher_enc_custom, + submit_cipher_enc_custom, + submit_cipher_enc_custom, + /* [7] DES */ + submit_cipher_enc_des_cbc_64, + submit_cipher_enc_des_cbc_64, + submit_cipher_enc_des_cbc_64, + submit_cipher_enc_des_cbc_64, + /* [8] DOCSIS DES */ + submit_cipher_enc_des_docsis_64, + submit_cipher_enc_des_docsis_64, + submit_cipher_enc_des_docsis_64, + submit_cipher_enc_des_docsis_64, + /* [9] AES-CCM */ + submit_cipher_enc_null, + submit_cipher_enc_aes_ccm_128, + submit_cipher_enc_null, + submit_cipher_enc_aes_ccm_256, + /* [10] 3DES */ + submit_cipher_enc_des3_cbc_192, + submit_cipher_enc_des3_cbc_192, + submit_cipher_enc_des3_cbc_192, + submit_cipher_enc_des3_cbc_192, + /* [11] PON AES-CTR */ + submit_cipher_enc_aes_ctr_pon_128, + submit_cipher_enc_aes_ctr_pon_128, + submit_cipher_enc_aes_ctr_pon_128, + submit_cipher_enc_aes_ctr_pon_128, + /* [12] AES-ECB */ + submit_cipher_enc_null, + submit_cipher_enc_aes_ecb_128, + submit_cipher_enc_aes_ecb_192, + submit_cipher_enc_aes_ecb_256, + /* [13] AES-CTR BITLEN */ + submit_cipher_enc_null, + submit_cipher_enc_aes_ctr_128_bit, + submit_cipher_enc_aes_ctr_192_bit, + submit_cipher_enc_aes_ctr_256_bit, +#else /* __aarch64__ */ + /* [4] - [13] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [14] ZUC EEA3 */ + submit_cipher_enc_null, + submit_cipher_enc_zuc_eea3_128, + submit_cipher_enc_null, + submit_cipher_enc_zuc_eea3_256, + /* [15] SNOW3G UEA2 */ + submit_cipher_enc_snow3g_uea2_bit, + submit_cipher_enc_snow3g_uea2_bit, + submit_cipher_enc_snow3g_uea2_bit, + submit_cipher_enc_snow3g_uea2_bit, +#ifndef __aarch64__ + /* [16] KASUMI F8 UEA1 */ + submit_cipher_enc_kasumi_uea1_bit, + submit_cipher_enc_kasumi_uea1_bit, + submit_cipher_enc_kasumi_uea1_bit, + submit_cipher_enc_kasumi_uea1_bit, + /* [17] AES-CBCS-1-9 */ + submit_cipher_enc_aes_cbcs_1_9, + submit_cipher_enc_aes_cbcs_1_9, + submit_cipher_enc_aes_cbcs_1_9, + submit_cipher_enc_aes_cbcs_1_9, + /* [18] CHACHA20 */ + submit_cipher_enc_chacha20, + submit_cipher_enc_chacha20, + submit_cipher_enc_chacha20, + submit_cipher_enc_chacha20, + /* [19] CHACHA20-POLY1305 */ + submit_cipher_enc_chacha20_poly1305, + submit_cipher_enc_chacha20_poly1305, + submit_cipher_enc_chacha20_poly1305, + submit_cipher_enc_chacha20_poly1305, + /* [20] CHACHA20-POLY1305 SGL */ + submit_cipher_enc_chacha20_poly1305_sgl, + submit_cipher_enc_chacha20_poly1305_sgl, + submit_cipher_enc_chacha20_poly1305_sgl, + submit_cipher_enc_chacha20_poly1305_sgl, + /* [21] SNOW-V */ + submit_cipher_enc_snow_v, + submit_cipher_enc_snow_v, + submit_cipher_enc_snow_v, + submit_cipher_enc_snow_v, + /* [22] SNOW-V AEAD */ + submit_cipher_enc_snow_v_aead, + submit_cipher_enc_snow_v_aead, + submit_cipher_enc_snow_v_aead, + submit_cipher_enc_snow_v_aead, + /* [23] AES-GCM SGL */ + submit_cipher_enc_null, + submit_cipher_enc_aes_gcm_128_sgl, + submit_cipher_enc_aes_gcm_192_sgl, + submit_cipher_enc_aes_gcm_256_sgl, +#else /* __aarch64__ */ + /* [16] - [23] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + + /* add new cipher encrypt here */ + + /* [24] NULL */ + NULL, NULL, NULL, NULL, + /* [25] NULL */ + NULL, NULL, NULL, NULL, + /* [26] NULL */ + NULL, NULL, NULL, NULL, + /* [27] NULL */ + NULL, NULL, NULL, NULL, + /* [28] NULL */ + NULL, NULL, NULL, NULL, + /* [29] NULL */ + NULL, NULL, NULL, NULL, + /* [30] NULL */ + NULL, NULL, NULL, NULL, + /* [31] NULL */ + NULL, NULL, NULL, NULL, + +}; + +/* ========================================================================= */ +/* Generate specialized flush cipher functions and create a table */ +/* ========================================================================= */ + +/* ========================= */ +/* ======== DECRYPT ======== */ +/* ========================= */ + +#ifndef __aarch64__ +/* AES-CBC */ +static IMB_JOB *flush_cipher_dec_aes_cbc_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBC, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBC, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_cbc_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBC, + IMB_KEY_256_BYTES); +} + +/* AES-CTR */ +static IMB_JOB *flush_cipher_dec_aes_ctr_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_ctr_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_ctr_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* NULL */ +static IMB_JOB *flush_cipher_dec_null(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_NULL, + IMB_KEY_128_BYTES); +} + +#ifndef __aarch64__ +/* AES DOCSIS */ +static IMB_JOB *flush_cipher_dec_aes_docsis_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_docsis_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_256_BYTES); +} + +/* AES-GCM */ +static IMB_JOB *flush_cipher_dec_aes_gcm_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_gcm_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_gcm_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM, + IMB_KEY_256_BYTES); +} + +/* CUSTOM */ +static IMB_JOB *flush_cipher_dec_custom(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CUSTOM, + IMB_KEY_128_BYTES); +} + +/* DES */ +static IMB_JOB *flush_cipher_dec_des_cbc_64(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DES, + IMB_KEY_64_BYTES); +} + +/* DES DOCSIS */ +static IMB_JOB *flush_cipher_dec_des_docsis_64(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DOCSIS_DES, + IMB_KEY_64_BYTES); +} + +/* AES-CCM */ +static IMB_JOB *flush_cipher_dec_aes_ccm_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CCM, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_ccm_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CCM, + IMB_KEY_256_BYTES); +} + +/* 3DES */ +static IMB_JOB *flush_cipher_dec_des3_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_DES3, + IMB_KEY_192_BYTES); +} + +/* PON AES-CTR */ +static IMB_JOB *flush_cipher_dec_aes_ctr_pon_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_PON_AES_CNTR, + IMB_KEY_128_BYTES); +} + +/* AES-ECB */ +static IMB_JOB *flush_cipher_dec_aes_ecb_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ECB, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_ecb_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ECB, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_ecb_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ECB, + IMB_KEY_256_BYTES); +} + +/* AES-CTR BITS */ +static IMB_JOB *flush_cipher_dec_aes_ctr_128_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_ctr_192_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_ctr_256_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* ZUC EEA3 */ +static IMB_JOB *flush_cipher_dec_zuc_eea3_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_zuc_eea3_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_256_BYTES); +} + +/* SNOW3G UEA2 */ +static IMB_JOB *flush_cipher_dec_snow3g_uea2_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_SNOW3G_UEA2_BITLEN, + IMB_KEY_128_BYTES); +} + +#ifndef __aarch64__ +/* KASUMI F8 UEA1 */ +static IMB_JOB *flush_cipher_dec_kasumi_uea1_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_KASUMI_UEA1_BITLEN, + IMB_KEY_128_BYTES); +} + +/* AES-CBCS-1-9 */ +static IMB_JOB *flush_cipher_dec_aes_cbcs_1_9(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CBCS_1_9, + IMB_KEY_128_BYTES); +} + +/* CHACHA20 */ +static IMB_JOB *flush_cipher_dec_chacha20(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CHACHA20, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 */ +static IMB_JOB * +flush_cipher_dec_chacha20_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_CHACHA20_POLY1305, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 SGL */ +static IMB_JOB * +flush_cipher_dec_chacha20_poly1305_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, + IMB_CIPHER_CHACHA20_POLY1305_SGL, + IMB_KEY_256_BYTES); +} + +/* SNOW-V */ +static IMB_JOB *flush_cipher_dec_snow_v(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_SNOW_V, + IMB_KEY_256_BYTES); +} + +/* SNOW-V AEAD */ +static IMB_JOB *flush_cipher_dec_snow_v_aead(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_SNOW_V_AEAD, + IMB_KEY_256_BYTES); +} + +/* AES-GCM SGL */ +static IMB_JOB *flush_cipher_dec_aes_gcm_128_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_gcm_192_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_dec_aes_gcm_256_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_DEC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* ========================= */ +/* ======== ENCRYPT ======== */ +/* ========================= */ + +#ifndef __aarch64__ +/* AES-CBC */ +static IMB_JOB *flush_cipher_enc_aes_cbc_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBC, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBC, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_cbc_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBC, + IMB_KEY_256_BYTES); +} + +/* AES-CTR */ +static IMB_JOB *flush_cipher_enc_aes_ctr_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_ctr_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_ctr_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* NULL */ +static IMB_JOB *flush_cipher_enc_null(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_NULL, + IMB_KEY_128_BYTES); +} + +#ifndef __aarch64__ +/* AES DOCSIS */ +static IMB_JOB *flush_cipher_enc_aes_docsis_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_docsis_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DOCSIS_SEC_BPI, + IMB_KEY_256_BYTES); +} + +/* AES-GCM */ +static IMB_JOB *flush_cipher_enc_aes_gcm_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_gcm_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_gcm_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM, + IMB_KEY_256_BYTES); +} + +/* CUSTOM */ +static IMB_JOB *flush_cipher_enc_custom(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CUSTOM, + IMB_KEY_128_BYTES); +} + +/* DES */ +static IMB_JOB *flush_cipher_enc_des_cbc_64(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DES, + IMB_KEY_64_BYTES); +} + +/* DES DOCSIS */ +static IMB_JOB *flush_cipher_enc_des_docsis_64(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DOCSIS_DES, + IMB_KEY_64_BYTES); +} + +/* AES-CCM */ +static IMB_JOB *flush_cipher_enc_aes_ccm_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CCM, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_ccm_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CCM, + IMB_KEY_256_BYTES); +} + +/* 3DES */ +static IMB_JOB *flush_cipher_enc_des3_cbc_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_DES3, + IMB_KEY_192_BYTES); +} + +/* PON AES-CTR */ +static IMB_JOB *flush_cipher_enc_aes_ctr_pon_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_PON_AES_CNTR, + IMB_KEY_128_BYTES); +} + +/* AES-ECB */ +static IMB_JOB *flush_cipher_enc_aes_ecb_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ECB, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_ecb_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ECB, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_ecb_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ECB, + IMB_KEY_256_BYTES); +} + +/* AES-CTR BITS */ +static IMB_JOB *flush_cipher_enc_aes_ctr_128_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_ctr_192_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_ctr_256_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CNTR_BITLEN, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* ZUC EEA3 */ +static IMB_JOB *flush_cipher_enc_zuc_eea3_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_zuc_eea3_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_ZUC_EEA3, + IMB_KEY_256_BYTES); +} + +/* SNOW3G UEA2 */ +static IMB_JOB *flush_cipher_enc_snow3g_uea2_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_SNOW3G_UEA2_BITLEN, + IMB_KEY_128_BYTES); +} + +#ifndef __aarch64__ +/* KASUMI F8 UEA1 */ +static IMB_JOB *flush_cipher_enc_kasumi_uea1_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_KASUMI_UEA1_BITLEN, + IMB_KEY_128_BYTES); +} + +/* AES-CBCS-1-9 */ +static IMB_JOB *flush_cipher_enc_aes_cbcs_1_9(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CBCS_1_9, + IMB_KEY_128_BYTES); +} + +/* CHACHA20 */ +static IMB_JOB *flush_cipher_enc_chacha20(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CHACHA20, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 */ +static IMB_JOB * +flush_cipher_enc_chacha20_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_CHACHA20_POLY1305, + IMB_KEY_256_BYTES); +} + +/* CHACHA20-POLY1305 SGL */ +static IMB_JOB * +flush_cipher_enc_chacha20_poly1305_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, + IMB_CIPHER_CHACHA20_POLY1305_SGL, + IMB_KEY_256_BYTES); +} + +/* SNOW-V */ +static IMB_JOB *flush_cipher_enc_snow_v(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_SNOW_V, + IMB_KEY_256_BYTES); +} + +/* SNOW-V AEAD */ +static IMB_JOB *flush_cipher_enc_snow_v_aead(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_SNOW_V_AEAD, + IMB_KEY_256_BYTES); +} + +/* AES-GCM SGL */ +static IMB_JOB *flush_cipher_enc_aes_gcm_128_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_128_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_gcm_192_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_192_BYTES); +} + +static IMB_JOB *flush_cipher_enc_aes_gcm_256_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_CIPHER_ENC(state, job, IMB_CIPHER_GCM_SGL, + IMB_KEY_256_BYTES); +} +#endif /* __aarch64__ */ + +/* + * Four entries per algorithm (different key sizes), + * algorithms in the same order IMB_CIPHER_MODE + * index 0 - key size from 0 to 64-bits + * index 1 - key size from 65 to 128-bits + * index 2 - key size from 129 to 192-bits + * index 3 - key size from 193 to 256-bits + */ +static const submit_flush_fn_t tab_flush_cipher[] = { + /* ========================= */ + /* === DECRYPT DIRECTION === */ + /* ========================= */ + + /* [0] keep empty - enums start from value 1 */ + NULL, NULL, NULL, NULL, +#ifndef __aarch64__ + /* [1] AES-CBC */ + flush_cipher_dec_null, + flush_cipher_dec_aes_cbc_128, + flush_cipher_dec_aes_cbc_192, + flush_cipher_dec_aes_cbc_256, + /* [2] AES-CBC */ + flush_cipher_dec_null, + flush_cipher_dec_aes_ctr_128, + flush_cipher_dec_aes_ctr_192, + flush_cipher_dec_aes_ctr_256, +#else /* __aarch64__ */ + /* [1] - [2] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [3] NULL */ + flush_cipher_dec_null, + flush_cipher_dec_null, + flush_cipher_dec_null, + flush_cipher_dec_null, +#ifndef __aarch64__ + /* [4] DOCSIS SEC BPI */ + flush_cipher_dec_null, + flush_cipher_dec_aes_docsis_128, + flush_cipher_dec_null, + flush_cipher_dec_aes_docsis_256, + /* [5] AES-GCM */ + flush_cipher_dec_null, + flush_cipher_dec_aes_gcm_128, + flush_cipher_dec_aes_gcm_192, + flush_cipher_dec_aes_gcm_256, + /* [6] CUSTOM */ + flush_cipher_dec_custom, + flush_cipher_dec_custom, + flush_cipher_dec_custom, + flush_cipher_dec_custom, + /* [7] DES */ + flush_cipher_dec_des_cbc_64, + flush_cipher_dec_des_cbc_64, + flush_cipher_dec_des_cbc_64, + flush_cipher_dec_des_cbc_64, + /* [8] DOCSIS DES */ + flush_cipher_dec_des_docsis_64, + flush_cipher_dec_des_docsis_64, + flush_cipher_dec_des_docsis_64, + flush_cipher_dec_des_docsis_64, + /* [9] AES-CCM */ + flush_cipher_dec_null, + flush_cipher_dec_aes_ccm_128, + flush_cipher_dec_null, + flush_cipher_dec_aes_ccm_256, + /* [10] 3DES */ + flush_cipher_dec_des3_cbc_192, + flush_cipher_dec_des3_cbc_192, + flush_cipher_dec_des3_cbc_192, + flush_cipher_dec_des3_cbc_192, + /* [11] PON AES-CTR */ + flush_cipher_dec_aes_ctr_pon_128, + flush_cipher_dec_aes_ctr_pon_128, + flush_cipher_dec_aes_ctr_pon_128, + flush_cipher_dec_aes_ctr_pon_128, + /* [12] AES-ECB */ + flush_cipher_dec_null, + flush_cipher_dec_aes_ecb_128, + flush_cipher_dec_aes_ecb_192, + flush_cipher_dec_aes_ecb_256, + /* [13] AES-CTR BITLEN */ + flush_cipher_dec_null, + flush_cipher_dec_aes_ctr_128_bit, + flush_cipher_dec_aes_ctr_192_bit, + flush_cipher_dec_aes_ctr_256_bit, +#else /* __aarch64__ */ + /* [4] - [13] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [14] ZUC EEA3 */ + flush_cipher_dec_null, + flush_cipher_dec_zuc_eea3_128, + flush_cipher_dec_null, + flush_cipher_dec_zuc_eea3_256, + /* [15] SNOW3G UEA2 */ + flush_cipher_dec_snow3g_uea2_bit, + flush_cipher_dec_snow3g_uea2_bit, + flush_cipher_dec_snow3g_uea2_bit, + flush_cipher_dec_snow3g_uea2_bit, +#ifndef __aarch64__ + /* [16] KASUMI F8 UEA1 */ + flush_cipher_dec_kasumi_uea1_bit, + flush_cipher_dec_kasumi_uea1_bit, + flush_cipher_dec_kasumi_uea1_bit, + flush_cipher_dec_kasumi_uea1_bit, + /* [17] AES-CBCS-1-9 */ + flush_cipher_dec_aes_cbcs_1_9, + flush_cipher_dec_aes_cbcs_1_9, + flush_cipher_dec_aes_cbcs_1_9, + flush_cipher_dec_aes_cbcs_1_9, + /* [18] CHACHA20 */ + flush_cipher_dec_chacha20, + flush_cipher_dec_chacha20, + flush_cipher_dec_chacha20, + flush_cipher_dec_chacha20, + /* [19] CHACHA20-POLY1305 */ + flush_cipher_dec_chacha20_poly1305, + flush_cipher_dec_chacha20_poly1305, + flush_cipher_dec_chacha20_poly1305, + flush_cipher_dec_chacha20_poly1305, + /* [20] CHACHA20-POLY1305 SGL */ + flush_cipher_dec_chacha20_poly1305_sgl, + flush_cipher_dec_chacha20_poly1305_sgl, + flush_cipher_dec_chacha20_poly1305_sgl, + flush_cipher_dec_chacha20_poly1305_sgl, + /* [21] SNOW-V */ + flush_cipher_dec_snow_v, + flush_cipher_dec_snow_v, + flush_cipher_dec_snow_v, + flush_cipher_dec_snow_v, + /* [22] SNOW-V AEAD */ + flush_cipher_dec_snow_v_aead, + flush_cipher_dec_snow_v_aead, + flush_cipher_dec_snow_v_aead, + flush_cipher_dec_snow_v_aead, + /* [23] AES-GCM SGL */ + flush_cipher_dec_null, + flush_cipher_dec_aes_gcm_128_sgl, + flush_cipher_dec_aes_gcm_192_sgl, + flush_cipher_dec_aes_gcm_256_sgl, +#else /* __aarch64__ */ + /* [16] - [23] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + + /* add new cipher decrypt here */ + + /* [24] NULL */ + NULL, NULL, NULL, NULL, + /* [25] NULL */ + NULL, NULL, NULL, NULL, + /* [26] NULL */ + NULL, NULL, NULL, NULL, + /* [27] NULL */ + NULL, NULL, NULL, NULL, + /* [28] NULL */ + NULL, NULL, NULL, NULL, + /* [29] NULL */ + NULL, NULL, NULL, NULL, + /* [30] NULL */ + NULL, NULL, NULL, NULL, + /* [31] NULL */ + NULL, NULL, NULL, NULL, + + /* ========================= */ + /* === ENCRYPT DIRECTION === */ + /* ========================= */ + + /* [0] keep empty - enums start from value 1 */ + NULL, NULL, NULL, NULL, +#ifndef __aarch64__ + /* [1] AES-CBC */ + flush_cipher_enc_null, + flush_cipher_enc_aes_cbc_128, + flush_cipher_enc_aes_cbc_192, + flush_cipher_enc_aes_cbc_256, + /* [2] AES-CBC */ + flush_cipher_enc_null, + flush_cipher_enc_aes_ctr_128, + flush_cipher_enc_aes_ctr_192, + flush_cipher_enc_aes_ctr_256, +#else /* __aarch64__ */ + /* [1] - [2] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [3] NULL */ + flush_cipher_enc_null, + flush_cipher_enc_null, + flush_cipher_enc_null, + flush_cipher_enc_null, +#ifndef __aarch64__ + /* [4] DOCSIS SEC BPI */ + flush_cipher_enc_null, + flush_cipher_enc_aes_docsis_128, + flush_cipher_enc_null, + flush_cipher_enc_aes_docsis_256, + /* [5] AES-GCM */ + flush_cipher_enc_null, + flush_cipher_enc_aes_gcm_128, + flush_cipher_enc_aes_gcm_192, + flush_cipher_enc_aes_gcm_256, + /* [6] CUSTOM */ + flush_cipher_enc_custom, + flush_cipher_enc_custom, + flush_cipher_enc_custom, + flush_cipher_enc_custom, + /* [7] DES */ + flush_cipher_enc_des_cbc_64, + flush_cipher_enc_des_cbc_64, + flush_cipher_enc_des_cbc_64, + flush_cipher_enc_des_cbc_64, + /* [8] DOCSIS DES */ + flush_cipher_enc_des_docsis_64, + flush_cipher_enc_des_docsis_64, + flush_cipher_enc_des_docsis_64, + flush_cipher_enc_des_docsis_64, + /* [9] AES-CCM */ + flush_cipher_enc_null, + flush_cipher_enc_aes_ccm_128, + flush_cipher_enc_null, + flush_cipher_enc_aes_ccm_256, + /* [10] 3DES */ + flush_cipher_enc_des3_cbc_192, + flush_cipher_enc_des3_cbc_192, + flush_cipher_enc_des3_cbc_192, + flush_cipher_enc_des3_cbc_192, + /* [11] PON AES-CTR */ + flush_cipher_enc_aes_ctr_pon_128, + flush_cipher_enc_aes_ctr_pon_128, + flush_cipher_enc_aes_ctr_pon_128, + flush_cipher_enc_aes_ctr_pon_128, + /* [12] AES-ECB */ + flush_cipher_enc_null, + flush_cipher_enc_aes_ecb_128, + flush_cipher_enc_aes_ecb_192, + flush_cipher_enc_aes_ecb_256, + /* [13] AES-CTR BITLEN */ + flush_cipher_enc_null, + flush_cipher_enc_aes_ctr_128_bit, + flush_cipher_enc_aes_ctr_192_bit, + flush_cipher_enc_aes_ctr_256_bit, +#else /* __aarch64__ */ + /* [4] - [13] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [14] ZUC EEA3 */ + flush_cipher_enc_null, + flush_cipher_enc_zuc_eea3_128, + flush_cipher_enc_null, + flush_cipher_enc_zuc_eea3_256, + /* [15] SNOW3G UEA2 */ + flush_cipher_enc_snow3g_uea2_bit, + flush_cipher_enc_snow3g_uea2_bit, + flush_cipher_enc_snow3g_uea2_bit, + flush_cipher_enc_snow3g_uea2_bit, +#ifndef __aarch64__ + /* [16] KASUMI F8 UEA1 */ + flush_cipher_enc_kasumi_uea1_bit, + flush_cipher_enc_kasumi_uea1_bit, + flush_cipher_enc_kasumi_uea1_bit, + flush_cipher_enc_kasumi_uea1_bit, + /* [17] AES-CBCS-1-9 */ + flush_cipher_enc_aes_cbcs_1_9, + flush_cipher_enc_aes_cbcs_1_9, + flush_cipher_enc_aes_cbcs_1_9, + flush_cipher_enc_aes_cbcs_1_9, + /* [18] CHACHA20 */ + flush_cipher_enc_chacha20, + flush_cipher_enc_chacha20, + flush_cipher_enc_chacha20, + flush_cipher_enc_chacha20, + /* [19] CHACHA20-POLY1305 */ + flush_cipher_enc_chacha20_poly1305, + flush_cipher_enc_chacha20_poly1305, + flush_cipher_enc_chacha20_poly1305, + flush_cipher_enc_chacha20_poly1305, + /* [20] CHACHA20-POLY1305 SGL */ + flush_cipher_enc_chacha20_poly1305_sgl, + flush_cipher_enc_chacha20_poly1305_sgl, + flush_cipher_enc_chacha20_poly1305_sgl, + flush_cipher_enc_chacha20_poly1305_sgl, + /* [21] SNOW-V */ + flush_cipher_enc_snow_v, + flush_cipher_enc_snow_v, + flush_cipher_enc_snow_v, + flush_cipher_enc_snow_v, + /* [22] SNOW-V AEAD */ + flush_cipher_enc_snow_v_aead, + flush_cipher_enc_snow_v_aead, + flush_cipher_enc_snow_v_aead, + flush_cipher_enc_snow_v_aead, + /* [23] AES-GCM SGL */ + flush_cipher_enc_null, + flush_cipher_enc_aes_gcm_128_sgl, + flush_cipher_enc_aes_gcm_192_sgl, + flush_cipher_enc_aes_gcm_256_sgl, +#else /* __aarch64__ */ + /* [16] - [23] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + + /* add new cipher encrypt here */ + + /* [24] NULL */ + NULL, NULL, NULL, NULL, + /* [25] NULL */ + NULL, NULL, NULL, NULL, + /* [26] NULL */ + NULL, NULL, NULL, NULL, + /* [27] NULL */ + NULL, NULL, NULL, NULL, + /* [28] NULL */ + NULL, NULL, NULL, NULL, + /* [29] NULL */ + NULL, NULL, NULL, NULL, + /* [30] NULL */ + NULL, NULL, NULL, NULL, + /* [31] NULL */ + NULL, NULL, NULL, NULL, + +}; + +/* ========================================================================= */ +/* Hash submit & flush functions */ +/* ========================================================================= */ + +__forceinline +IMB_JOB * +SUBMIT_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) +{ +#ifndef __aarch64__ + MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; + MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; + MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; + MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; + MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; + MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; + MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; +#endif /* __aarch64__ */ + MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; + MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; +#ifndef __aarch64__ + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; + MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; + MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; +#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) + MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; +#endif +#else /* __aarch64__ */ + MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; +#endif /* __aarch64__ */ + + switch (hash_alg) { +#ifndef __aarch64__ + case IMB_AUTH_HMAC_SHA_1: + return SUBMIT_JOB_HMAC(hmac_sha_1_ooo, job); + case IMB_AUTH_HMAC_SHA_224: + return SUBMIT_JOB_HMAC_SHA_224(hmac_sha_224_ooo, job); + case IMB_AUTH_HMAC_SHA_256: + return SUBMIT_JOB_HMAC_SHA_256(hmac_sha_256_ooo, job); + case IMB_AUTH_HMAC_SHA_384: + return SUBMIT_JOB_HMAC_SHA_384(hmac_sha_384_ooo, job); + case IMB_AUTH_HMAC_SHA_512: + return SUBMIT_JOB_HMAC_SHA_512(hmac_sha_512_ooo, job); + case IMB_AUTH_AES_XCBC: + return SUBMIT_JOB_AES_XCBC(aes_xcbc_ooo, job); + case IMB_AUTH_MD5: + return SUBMIT_JOB_HMAC_MD5(hmac_md5_ooo, job); + case IMB_AUTH_CUSTOM: + return SUBMIT_JOB_CUSTOM_HASH(job); + case IMB_AUTH_AES_CCM: + if (16 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES128_CCM_AUTH(aes_ccm_ooo, job); + } else { /* assume 32 */ + return SUBMIT_JOB_AES256_CCM_AUTH(aes256_ccm_ooo, job); + } + case IMB_AUTH_AES_CMAC: + /* + * CMAC OOO MGR assumes job len in bits + * (for CMAC length is provided in bytes) + */ + job->msg_len_to_hash_in_bits = + job->msg_len_to_hash_in_bytes * 8; + return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); + case IMB_AUTH_AES_CMAC_BITLEN: + return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); + case IMB_AUTH_AES_CMAC_256: + job->msg_len_to_hash_in_bits = + job->msg_len_to_hash_in_bytes * 8; + return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); + case IMB_AUTH_SHA_1: + return SUBMIT_JOB_SHA1(sha_1_ooo, job); + case IMB_AUTH_SHA_224: + return SUBMIT_JOB_SHA224(sha_224_ooo, job); + case IMB_AUTH_SHA_256: + return SUBMIT_JOB_SHA256(sha_256_ooo, job); + case IMB_AUTH_SHA_384: + return SUBMIT_JOB_SHA384(sha_384_ooo, job); + case IMB_AUTH_SHA_512: + return SUBMIT_JOB_SHA512(sha_512_ooo, job); +#endif /* __aarch64__ */ + case IMB_AUTH_ZUC_EIA3_BITLEN: + return SUBMIT_JOB_ZUC_EIA3(zuc_eia3_ooo, job); + case IMB_AUTH_ZUC256_EIA3_BITLEN: + return SUBMIT_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job, + job->auth_tag_output_len_in_bytes); + case IMB_AUTH_SNOW3G_UIA2_BITLEN: +#ifndef __aarch64__ +#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) + return SUBMIT_JOB_SNOW3G_UIA2(snow3g_uia2_ooo, job); +#else + IMB_SNOW3G_F9_1_BUFFER(state, (const snow3g_key_schedule_t *) + job->u.SNOW3G_UIA2._key, + job->u.SNOW3G_UIA2._iv, + job->src + job->hash_start_src_offset_in_bytes, + job->msg_len_to_hash_in_bits, + job->auth_tag_output); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; +#endif +#else /* __aarch64__ */ + return SUBMIT_JOB_SNOW3G_UIA2(snow3g_uia2_ooo, job); +#endif /* __aarch64__ */ + +#ifndef __aarch64__ + case IMB_AUTH_KASUMI_UIA1: + IMB_KASUMI_F9_1_BUFFER(state, (const kasumi_key_sched_t *) + job->u.KASUMI_UIA1._key, + job->src + job->hash_start_src_offset_in_bytes, + (const uint32_t) job->msg_len_to_hash_in_bytes, + job->auth_tag_output); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_AES_GMAC_128: + process_gmac(state, job, IMB_KEY_128_BYTES); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_AES_GMAC_192: + process_gmac(state, job, IMB_KEY_192_BYTES); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_AES_GMAC_256: + process_gmac(state, job, IMB_KEY_256_BYTES); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_POLY1305: + POLY1305_MAC(job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC32_ETHERNET_FCS: + CRC(IMB_CRC32_ETHERNET_FCS, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC32_SCTP: + CRC(IMB_CRC32_SCTP, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: + CRC(IMB_CRC32_WIMAX_OFDMA_DATA, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC24_LTE_A: + CRC(IMB_CRC24_LTE_A, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC24_LTE_B: + CRC(IMB_CRC24_LTE_B, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC16_X25: + CRC(IMB_CRC16_X25, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC16_FP_DATA: + CRC(IMB_CRC16_FP_DATA, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC11_FP_HEADER: + CRC(IMB_CRC11_FP_HEADER, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC10_IUUP_DATA: + CRC(IMB_CRC10_IUUP_DATA, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: + CRC(IMB_CRC8_WIMAX_OFDMA_HCS, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC7_FP_HEADER: + CRC(IMB_CRC7_FP_HEADER, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC6_IUUP_HEADER: + CRC(IMB_CRC6_IUUP_HEADER, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_GHASH: + return process_ghash(state, job); +#endif /* __aarch64__ */ + default: + /** + * assume IMB_AUTH_GCM, IMB_AUTH_PON_CRC_BIP, + * IMB_AUTH_SNOW_V_AEAD or IMB_AUTH_NULL + */ + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + } +} + +__forceinline +IMB_JOB * +FLUSH_JOB_HASH_EX(IMB_MGR *state, IMB_JOB *job, const IMB_HASH_ALG hash_alg) +{ +#ifndef __aarch64__ + MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; + MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; + MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; + MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; + MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; + MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; + MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; +#endif /* __aarch64__ */ + MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; + MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; +#ifndef __aarch64__ + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; + MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; + MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; +#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) + MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; +#endif +#else /* __aarch64__ */ + MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; +#endif /* __aarch64__ */ + + switch (hash_alg) { +#ifndef __aarch64__ + case IMB_AUTH_HMAC_SHA_1: + return FLUSH_JOB_HMAC(hmac_sha_1_ooo); + case IMB_AUTH_HMAC_SHA_224: + return FLUSH_JOB_HMAC_SHA_224(hmac_sha_224_ooo); + case IMB_AUTH_HMAC_SHA_256: + return FLUSH_JOB_HMAC_SHA_256(hmac_sha_256_ooo); + case IMB_AUTH_HMAC_SHA_384: + return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); + case IMB_AUTH_HMAC_SHA_512: + return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); + case IMB_AUTH_SHA_1: + return FLUSH_JOB_SHA1(sha_1_ooo, job); + case IMB_AUTH_SHA_224: + return FLUSH_JOB_SHA224(sha_224_ooo, job); + case IMB_AUTH_SHA_256: + return FLUSH_JOB_SHA256(sha_256_ooo, job); + case IMB_AUTH_SHA_384: + return FLUSH_JOB_SHA384(sha_384_ooo, job); + case IMB_AUTH_SHA_512: + return FLUSH_JOB_SHA512(sha_512_ooo, job); + case IMB_AUTH_AES_XCBC: + return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); + case IMB_AUTH_MD5: + return FLUSH_JOB_HMAC_MD5(hmac_md5_ooo); + case IMB_AUTH_CUSTOM: + return FLUSH_JOB_CUSTOM_HASH(job); + case IMB_AUTH_AES_CCM: + if (16 == job->key_len_in_bytes) { + return FLUSH_JOB_AES128_CCM_AUTH(aes_ccm_ooo); + } else { /* assume 32 */ + return FLUSH_JOB_AES256_CCM_AUTH(aes256_ccm_ooo); + } case IMB_AUTH_AES_CMAC: case IMB_AUTH_AES_CMAC_BITLEN: return FLUSH_JOB_AES128_CMAC_AUTH(aes_cmac_ooo); case IMB_AUTH_AES_CMAC_256: return FLUSH_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo); +#endif /* __aarch64__ */ case IMB_AUTH_ZUC_EIA3_BITLEN: return FLUSH_JOB_ZUC_EIA3(zuc_eia3_ooo); case IMB_AUTH_ZUC256_EIA3_BITLEN: return FLUSH_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job->auth_tag_output_len_in_bytes); +#ifndef __aarch64__ #if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) case IMB_AUTH_SNOW3G_UIA2_BITLEN: return FLUSH_JOB_SNOW3G_UIA2(snow3g_uia2_ooo); #endif +#else /* __aarch64__ */ + case IMB_AUTH_SNOW3G_UIA2_BITLEN: + return FLUSH_JOB_SNOW3G_UIA2(snow3g_uia2_ooo); +#endif /* __aarch64__ */ default: /* assume GCM or IMB_AUTH_NULL */ if (!(job->status & IMB_STATUS_COMPLETED_AUTH)) { job->status |= IMB_STATUS_COMPLETED_AUTH; @@ -998,30 +2754,797 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) } } +/* ========================================================================= */ +/* Generate specialized hash submit functions and create a table */ +/* ========================================================================= */ + +#ifndef __aarch64__ +static IMB_JOB *submit_hash_hmac_sha1(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_1); +} + +static IMB_JOB *submit_hash_hmac_sha224(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_224); +} + +static IMB_JOB *submit_hash_hmac_sha256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_256); +} + +static IMB_JOB *submit_hash_hmac_sha384(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_384); +} + +static IMB_JOB *submit_hash_hmac_sha512(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_512); +} + +static IMB_JOB *submit_hash_aes_xcbc(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_XCBC); +} + +static IMB_JOB *submit_hash_hmac_md5(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_MD5); +} +#endif /* __aarch64__ */ + +static IMB_JOB *submit_hash_null(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_NULL); +} + +#ifndef __aarch64__ +static IMB_JOB *submit_hash_aes_gmac(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC); +} + +static IMB_JOB *submit_hash_custom(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CUSTOM); +} + +static IMB_JOB *submit_hash_aes_ccm(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_CCM); +} + +static IMB_JOB *submit_hash_aes_cmac(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC); +} + +static IMB_JOB *submit_hash_sha1(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SHA_1); +} + +static IMB_JOB *submit_hash_sha224(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SHA_224); +} + +static IMB_JOB *submit_hash_sha256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SHA_256); +} + +static IMB_JOB *submit_hash_sha384(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SHA_384); +} + +static IMB_JOB *submit_hash_sha512(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SHA_512); +} + +static IMB_JOB *submit_hash_aes_cmac_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC_BITLEN); +} + +static IMB_JOB *submit_hash_pon_crc_bip(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_PON_CRC_BIP); +} +#endif /* __aarch64__ */ + +static IMB_JOB *submit_hash_zuc_eia3_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_ZUC_EIA3_BITLEN); +} + +#ifndef __aarch64__ +static IMB_JOB *submit_hash_docsis_crc32(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_DOCSIS_CRC32); +} +#endif /* __aarch64__ */ + +static IMB_JOB *submit_hash_snow3g_uia2_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SNOW3G_UIA2_BITLEN); +} + +#ifndef __aarch64__ +static IMB_JOB *submit_hash_kasumi_uia1(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_KASUMI_UIA1); +} + +static IMB_JOB *submit_hash_aes_gmac_128(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC_128); +} + +static IMB_JOB *submit_hash_aes_gmac_192(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC_192); +} + +static IMB_JOB *submit_hash_aes_gmac_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC_256); +} + +static IMB_JOB *submit_hash_aes_cmac_256(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC_256); +} + +static IMB_JOB *submit_hash_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_POLY1305); +} + +static IMB_JOB *submit_hash_chacha20_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CHACHA20_POLY1305); +} + +static IMB_JOB *submit_hash_chacha20_poly1305_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CHACHA20_POLY1305_SGL); +} +#endif /* __aarch64__ */ + +static IMB_JOB *submit_hash_zuc256_eia3_bit(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_ZUC256_EIA3_BITLEN); +} + +#ifndef __aarch64__ +static IMB_JOB *submit_hash_snow_v_aead(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_SNOW_V_AEAD); +} + +static IMB_JOB *submit_hash_gcm_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_GCM_SGL); +} + +static IMB_JOB *submit_hash_crc32_ethernet_fcs(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC32_ETHERNET_FCS); +} + +static IMB_JOB *submit_hash_crc32_sctp(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC32_SCTP); +} + +static IMB_JOB *submit_hash_crc32_wimax_ofdma(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC32_WIMAX_OFDMA_DATA); +} + +static IMB_JOB *submit_hash_crc24_lte_a(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC24_LTE_A); +} + +static IMB_JOB *submit_hash_crc24_lte_b(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC24_LTE_B); +} + +static IMB_JOB *submit_hash_crc16_x25(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC16_X25); +} + +static IMB_JOB *submit_hash_crc16_fp_data(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC16_FP_DATA); +} + +static IMB_JOB *submit_hash_crc11_fp_header(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC11_FP_HEADER); +} + +static IMB_JOB *submit_hash_crc10_iuup_data(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC10_IUUP_DATA); +} + +static IMB_JOB *submit_hash_crc8_wimax_odma(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC8_WIMAX_OFDMA_HCS); +} + +static IMB_JOB *submit_hash_crc7_fp_header(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC7_FP_HEADER); +} + +static IMB_JOB *submit_hash_crc6_iuup_header(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_CRC6_IUUP_HEADER); +} + +static IMB_JOB *submit_hash_ghash(IMB_MGR *state, IMB_JOB *job) +{ + return SUBMIT_JOB_HASH_EX(state, job, IMB_AUTH_GHASH); +} +#endif /* __aarch64__ */ + +static const submit_flush_fn_t tab_submit_hash[] = { + /* [0] invalid entry */ + NULL, +#ifndef __aarch64__ + /* [1] HMAC-SHA1 */ + submit_hash_hmac_sha1, + /* [2] HMAC-SHA224 */ + submit_hash_hmac_sha224, + /* [3] HMAC-SHA256 */ + submit_hash_hmac_sha256, + /* [4] HMAC-SHA384 */ + submit_hash_hmac_sha384, + /* [5] HMAC-SHA512 */ + submit_hash_hmac_sha512, + /* [6] AES-XCBC */ + submit_hash_aes_xcbc, + /* [7] HMAC-MD5 */ + submit_hash_hmac_md5, +#else /* __aarch64__ */ + /* [1] - [7] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [8] NULL */ + submit_hash_null, +#ifndef __aarch64__ + /* [9] AES-GMAC */ + submit_hash_aes_gmac, + /* [10] CUSTOM */ + submit_hash_custom, + /* [11] AES-CCM */ + submit_hash_aes_ccm, + /* [12] AES-CMAC */ + submit_hash_aes_cmac, + /* [13] SHA1 */ + submit_hash_sha1, + /* [14] SHA224 */ + submit_hash_sha224, + /* [15] SHA256 */ + submit_hash_sha256, + /* [16] SHA384 */ + submit_hash_sha384, + /* [17] SHA512 */ + submit_hash_sha512, + /* [18] AES-CMAC BIT */ + submit_hash_aes_cmac_bit, + /* [19] PON CRC BIP */ + submit_hash_pon_crc_bip, +#else /* __aarch64__ */ + /* [9] - [19] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [20] ZUC EIA3 BIT */ + submit_hash_zuc_eia3_bit, +#ifndef __aarch64__ + /* [21] DOCSIS CRC32 */ + submit_hash_docsis_crc32, +#else /* __aarch64__ */ + /* [21] NULL */ + NULL, +#endif /* __aarch64__ */ + /* [22] SNOW3G UIA2 BIT */ + submit_hash_snow3g_uia2_bit, +#ifndef __aarch64__ + /* [23] KASUMI UIA1 */ + submit_hash_kasumi_uia1, + /* [24] AES-GMAC-128 */ + submit_hash_aes_gmac_128, + /* [25] AES-GMAC-192 */ + submit_hash_aes_gmac_192, + /* [26] AES-GMAC-256 */ + submit_hash_aes_gmac_256, + /* [27] AES-CMAC-256 */ + submit_hash_aes_cmac_256, + /* [28] POLY1305 */ + submit_hash_poly1305, + /* [29] CHACHA20-POLY1305 */ + submit_hash_chacha20_poly1305, + /* [30] CHACHA20-POLY1305 SGL */ + submit_hash_chacha20_poly1305_sgl, +#else /* __aarch64__ */ + /* [23] - [30] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [31] ZUC256 EIA3 */ + submit_hash_zuc256_eia3_bit, +#ifndef __aarch64__ + /* [32] SNOW-V AEAD */ + submit_hash_snow_v_aead, + /* [33] GCM SGL */ + submit_hash_gcm_sgl, + /* [34] CRC32 ETHERNET FCS */ + submit_hash_crc32_ethernet_fcs, + /* [35] CRC32 SCTP */ + submit_hash_crc32_sctp, + /* [36] CRC32 WIMAX OFDMA DATA */ + submit_hash_crc32_wimax_ofdma, + /* [37] CRC24 LTE A */ + submit_hash_crc24_lte_a, + /* [38] CRC24 LTE B */ + submit_hash_crc24_lte_b, + /* [39] CRC16 X25 */ + submit_hash_crc16_x25, + /* [40] CRC16 FP DATA */ + submit_hash_crc16_fp_data, + /* [41] CRC11 FP HEADER */ + submit_hash_crc11_fp_header, + /* [42] CRC10 IUUP DATA */ + submit_hash_crc10_iuup_data, + /* [43] CRC8 WIMAX OFDMA HCS */ + submit_hash_crc8_wimax_odma, + /* [44] CRC7 FP HEADER */ + submit_hash_crc7_fp_header, + /* [45] CRC6 IUUP HEADER */ + submit_hash_crc6_iuup_header, + /* [46] GHASH */ + submit_hash_ghash, +#else /* __aarch64__ */ + /* [32] - [46] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* add new hash algorithms here */ +}; + +/* ========================================================================= */ +/* Generate specialized hash flush functions and create a table */ +/* ========================================================================= */ + +#ifndef __aarch64__ +static IMB_JOB *flush_hash_hmac_sha1(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_1); +} + +static IMB_JOB *flush_hash_hmac_sha224(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_224); +} + +static IMB_JOB *flush_hash_hmac_sha256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_256); +} + +static IMB_JOB *flush_hash_hmac_sha384(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_384); +} + +static IMB_JOB *flush_hash_hmac_sha512(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_HMAC_SHA_512); +} + +static IMB_JOB *flush_hash_aes_xcbc(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_XCBC); +} + +static IMB_JOB *flush_hash_hmac_md5(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_MD5); +} +#endif /* __aarch64__ */ + +static IMB_JOB *flush_hash_null(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_NULL); +} + +#ifndef __aarch64__ +static IMB_JOB *flush_hash_aes_gmac(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC); +} + +static IMB_JOB *flush_hash_custom(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CUSTOM); +} + +static IMB_JOB *flush_hash_aes_ccm(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_CCM); +} + +static IMB_JOB *flush_hash_aes_cmac(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC); +} + +static IMB_JOB *flush_hash_sha1(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SHA_1); +} + +static IMB_JOB *flush_hash_sha224(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SHA_224); +} + +static IMB_JOB *flush_hash_sha256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SHA_256); +} + +static IMB_JOB *flush_hash_sha384(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SHA_384); +} + +static IMB_JOB *flush_hash_sha512(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SHA_512); +} + +static IMB_JOB *flush_hash_aes_cmac_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC_BITLEN); +} + +static IMB_JOB *flush_hash_pon_crc_bip(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_PON_CRC_BIP); +} +#endif /* __aarch64__ */ + +static IMB_JOB *flush_hash_zuc_eia3_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_ZUC_EIA3_BITLEN); +} + +#ifndef __aarch64__ +static IMB_JOB *flush_hash_docsis_crc32(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_DOCSIS_CRC32); +} +#endif /* __aarch64__ */ + +static IMB_JOB *flush_hash_snow3g_uia2_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SNOW3G_UIA2_BITLEN); +} + +#ifndef __aarch64__ +static IMB_JOB *flush_hash_kasumi_uia1(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_KASUMI_UIA1); +} + +static IMB_JOB *flush_hash_aes_gmac_128(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC_128); +} + +static IMB_JOB *flush_hash_aes_gmac_192(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC_192); +} + +static IMB_JOB *flush_hash_aes_gmac_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_GMAC_256); +} + +static IMB_JOB *flush_hash_aes_cmac_256(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_AES_CMAC_256); +} + +static IMB_JOB *flush_hash_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_POLY1305); +} + +static IMB_JOB *flush_hash_chacha20_poly1305(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CHACHA20_POLY1305); +} + +static IMB_JOB *flush_hash_chacha20_poly1305_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CHACHA20_POLY1305_SGL); +} +#endif /* __aarch64__ */ + +static IMB_JOB *flush_hash_zuc256_eia3_bit(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_ZUC256_EIA3_BITLEN); +} + +#ifndef __aarch64__ +static IMB_JOB *flush_hash_snow_v_aead(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_SNOW_V_AEAD); +} + +static IMB_JOB *flush_hash_gcm_sgl(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_GCM_SGL); +} + +static IMB_JOB *flush_hash_crc32_ethernet_fcs(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC32_ETHERNET_FCS); +} + +static IMB_JOB *flush_hash_crc32_sctp(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC32_SCTP); +} + +static IMB_JOB *flush_hash_crc32_wimax_ofdma(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC32_WIMAX_OFDMA_DATA); +} + +static IMB_JOB *flush_hash_crc24_lte_a(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC24_LTE_A); +} + +static IMB_JOB *flush_hash_crc24_lte_b(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC24_LTE_B); +} + +static IMB_JOB *flush_hash_crc16_x25(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC16_X25); +} + +static IMB_JOB *flush_hash_crc16_fp_data(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC16_FP_DATA); +} + +static IMB_JOB *flush_hash_crc11_fp_header(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC11_FP_HEADER); +} + +static IMB_JOB *flush_hash_crc10_iuup_data(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC10_IUUP_DATA); +} + +static IMB_JOB *flush_hash_crc8_wimax_odma(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC8_WIMAX_OFDMA_HCS); +} + +static IMB_JOB *flush_hash_crc7_fp_header(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC7_FP_HEADER); +} + +static IMB_JOB *flush_hash_crc6_iuup_header(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_CRC6_IUUP_HEADER); +} + +static IMB_JOB *flush_hash_ghash(IMB_MGR *state, IMB_JOB *job) +{ + return FLUSH_JOB_HASH_EX(state, job, IMB_AUTH_GHASH); +} +#endif /* __aarch64__ */ + +static const submit_flush_fn_t tab_flush_hash[] = { + /* [0] invalid entry */ + NULL, +#ifndef __aarch64__ + /* [1] HMAC-SHA1 */ + flush_hash_hmac_sha1, + /* [2] HMAC-SHA224 */ + flush_hash_hmac_sha224, + /* [3] HMAC-SHA256 */ + flush_hash_hmac_sha256, + /* [4] HMAC-SHA384 */ + flush_hash_hmac_sha384, + /* [5] HMAC-SHA512 */ + flush_hash_hmac_sha512, + /* [6] AES-XCBC */ + flush_hash_aes_xcbc, + /* [7] HMAC-MD5 */ + flush_hash_hmac_md5, +#else /* __aarch64__ */ + /* [1] - [7] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [8] NULL */ + flush_hash_null, +#ifndef __aarch64__ + /* [9] AES-GMAC */ + flush_hash_aes_gmac, + /* [10] CUSTOM */ + flush_hash_custom, + /* [11] AES-CCM */ + flush_hash_aes_ccm, + /* [12] AES-CMAC */ + flush_hash_aes_cmac, + /* [13] SHA1 */ + flush_hash_sha1, + /* [14] SHA224 */ + flush_hash_sha224, + /* [15] SHA256 */ + flush_hash_sha256, + /* [16] SHA384 */ + flush_hash_sha384, + /* [17] SHA512 */ + flush_hash_sha512, + /* [18] AES-CMAC BIT */ + flush_hash_aes_cmac_bit, + /* [19] PON CRC BIP */ + flush_hash_pon_crc_bip, +#else /* __aarch64__ */ + /* [9] - [19] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [20] ZUC EIA3 BIT */ + flush_hash_zuc_eia3_bit, +#ifndef __aarch64__ + /* [21] DOCSIS CRC32 */ + flush_hash_docsis_crc32, +#else /* __aarch64__ */ + /* [21] NULL */ + NULL, +#endif /* __aarch64__ */ + /* [22] SNOW3G UIA2 BIT */ + flush_hash_snow3g_uia2_bit, +#ifndef __aarch64__ + /* [23] KASUMI UIA1 */ + flush_hash_kasumi_uia1, + /* [24] AES-GMAC-128 */ + flush_hash_aes_gmac_128, + /* [25] AES-GMAC-192 */ + flush_hash_aes_gmac_192, + /* [26] AES-GMAC-256 */ + flush_hash_aes_gmac_256, + /* [27] AES-CMAC-256 */ + flush_hash_aes_cmac_256, + /* [28] POLY1305 */ + flush_hash_poly1305, + /* [29] CHACHA20-POLY1305 */ + flush_hash_chacha20_poly1305, + /* [30] CHACHA20-POLY1305 SGL */ + flush_hash_chacha20_poly1305_sgl, +#else /* __aarch64__ */ + /* [23] - [30] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* [31] ZUC256 EIA3 */ + flush_hash_zuc256_eia3_bit, +#ifndef __aarch64__ + /* [32] SNOW-V AEAD */ + flush_hash_snow_v_aead, + /* [33] GCM SGL */ + flush_hash_gcm_sgl, + /* [34] CRC32 ETHERNET FCS */ + flush_hash_crc32_ethernet_fcs, + /* [35] CRC32 SCTP */ + flush_hash_crc32_sctp, + /* [36] CRC32 WIMAX OFDMA DATA */ + flush_hash_crc32_wimax_ofdma, + /* [37] CRC24 LTE A */ + flush_hash_crc24_lte_a, + /* [38] CRC24 LTE B */ + flush_hash_crc24_lte_b, + /* [39] CRC16 X25 */ + flush_hash_crc16_x25, + /* [40] CRC16 FP DATA */ + flush_hash_crc16_fp_data, + /* [41] CRC11 FP HEADER */ + flush_hash_crc11_fp_header, + /* [42] CRC10 IUUP DATA */ + flush_hash_crc10_iuup_data, + /* [43] CRC8 WIMAX OFDMA HCS */ + flush_hash_crc8_wimax_odma, + /* [44] CRC7 FP HEADER */ + flush_hash_crc7_fp_header, + /* [45] CRC6 IUUP HEADER */ + flush_hash_crc6_iuup_header, + /* [46] GHASH */ + flush_hash_ghash, +#else /* __aarch64__ */ + /* [32] - [46] NULL */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, +#endif /* __aarch64__ */ + /* add new hash algorithms here */ +}; + +__forceinline IMB_JOB *SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) +{ + return tab_submit_hash[job->hash_alg](state, job); +} + +__forceinline IMB_JOB *FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) +{ + return tab_flush_hash[job->hash_alg](state, job); +} + /* ========================================================================= */ /* Job submit & flush functions */ /* ========================================================================= */ +__forceinline unsigned calc_cipher_tab_index(const IMB_JOB *job) +{ + /* + * See include/mb_mgr_job_api.h for cipher table organization + * - cipher_mode x 4, four key sizes per cipher mode + * - map key_len_in_bytes into 0, 1, 2 & 3 index values + * - encrypt_direction_bit x (ENCRYPT_DECRYPT_GAP x 4) + */ + return (job->cipher_mode << 2) + + (((job->key_len_in_bytes - 1) >> 3) & 3) + + ((job->cipher_direction & IMB_DIR_ENCRYPT) << 7); +} + __forceinline IMB_JOB *SUBMIT_JOB_CIPHER(IMB_MGR *state, IMB_JOB *job) { - if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = SUBMIT_JOB_CIPHER_ENC(state, job); - else - job = SUBMIT_JOB_CIPHER_DEC(state, job); + const unsigned idx = calc_cipher_tab_index(job); - return job; + IMB_ASSERT(ENCRYPT_DECRYPT_GAP >= IMB_CIPHER_NUM); + + return tab_submit_cipher[idx](state, job); } __forceinline IMB_JOB *FLUSH_JOB_CIPHER(IMB_MGR *state, IMB_JOB *job) { - if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = FLUSH_JOB_CIPHER_ENC(state, job); - else - job = FLUSH_JOB_CIPHER_DEC(state, job); + const unsigned idx = calc_cipher_tab_index(job); - return job; + return tab_flush_cipher[idx](state, job); } /* submit a half-completed job, based on the status */ @@ -1041,6 +3564,11 @@ IMB_JOB *RESUBMIT_JOB(IMB_MGR *state, IMB_JOB *job) __forceinline IMB_JOB *submit_new_job(IMB_MGR *state, IMB_JOB *job) { +#ifndef __aarch64__ + if (job->cipher_mode == IMB_CIPHER_GCM) + return SUBMIT_JOB_CIPHER(state, job); +#endif /* __aarch64__ */ + if (job->chain_order == IMB_ORDER_CIPHER_HASH) job = SUBMIT_JOB_CIPHER(state, job); else @@ -1102,11 +3630,13 @@ submit_job_and_check(IMB_MGR *state, const int run_check) } } +#ifndef __aarch64__ #ifndef LINUX DECLARE_ALIGNED(imb_uint128_t xmm_save[10], 16); SAVE_XMMS(xmm_save); #endif +#endif /* __aarch64__ */ job = JOBS(state, state->next_job); @@ -1153,9 +3683,11 @@ submit_job_and_check(IMB_MGR *state, const int run_check) ADV_JOBS(&state->earliest_job); exit: +#ifndef __aarch64__ #ifndef LINUX RESTORE_XMMS(xmm_save); #endif +#endif /* __aarch64__ */ return job; } @@ -1184,15 +3716,19 @@ FLUSH_JOB(IMB_MGR *state) } #endif IMB_JOB *job; +#ifndef __aarch64__ #ifndef LINUX DECLARE_ALIGNED(imb_uint128_t xmm_save[10], 16); #endif +#endif /* __aarch64__ */ if (state->earliest_job < 0) return NULL; /* empty */ +#ifndef __aarch64__ #ifndef LINUX SAVE_XMMS(xmm_save); #endif +#endif /* __aarch64__ */ job = JOBS(state, state->earliest_job); (void) complete_job(state, job); @@ -1201,12 +3737,106 @@ FLUSH_JOB(IMB_MGR *state) if (state->earliest_job == state->next_job) state->earliest_job = -1; /* becomes empty */ +#ifndef __aarch64__ #ifndef LINUX RESTORE_XMMS(xmm_save); #endif +#endif /* __aarch64__ */ return job; } +/* ========================================================================= */ +/* Async burst job submit & flush functions */ +/* ========================================================================= */ + +__forceinline void set_cipher_suite_id(IMB_JOB *job, void **id) +{ + const unsigned c_idx = calc_cipher_tab_index(job); + const unsigned h_idx = (unsigned) job->hash_alg; + + id[0] = (void *) tab_submit_cipher[c_idx]; + id[1] = (void *) tab_submit_hash[h_idx]; + id[2] = (void *) tab_flush_cipher[c_idx]; + id[3] = (void *) tab_flush_hash[h_idx]; +} + +#define CALL_SUBMIT_CIPHER(s, j) ((submit_flush_fn_t) (j)->suite_id[0])(s, j) +#define CALL_FLUSH_CIPHER(s, j) ((submit_flush_fn_t) (j)->suite_id[2])(s, j) +#define CALL_SUBMIT_HASH(s, j) ((submit_flush_fn_t) (j)->suite_id[1])(s, j) +#define CALL_FLUSH_HASH(s, j) ((submit_flush_fn_t) (j)->suite_id[3])(s, j) + +IMB_DLL_EXPORT void SET_SUITE_ID_FN(IMB_MGR *state, IMB_JOB *job) +{ + (void) state; + set_cipher_suite_id(job, job->suite_id); +} + +__forceinline +IMB_JOB *RESUBMIT_BURST_JOB(IMB_MGR *state, IMB_JOB *job) +{ + while (job != NULL && job->status < IMB_STATUS_COMPLETED) { + if (job->status == IMB_STATUS_COMPLETED_AUTH) + job = CALL_SUBMIT_CIPHER(state, job); + else /* assumed job->status = IMB_STATUS_COMPLETED_CIPHER */ + job = CALL_SUBMIT_HASH(state, job); + } + + return job; +} + +__forceinline +IMB_JOB *submit_new_burst_job(IMB_MGR *state, IMB_JOB *job) +{ +#ifndef __aarch64__ + if (job->cipher_mode == IMB_CIPHER_GCM) + return CALL_SUBMIT_CIPHER(state, job); +#endif /* __aarch64__ */ + + if (job->chain_order == IMB_ORDER_CIPHER_HASH) + job = CALL_SUBMIT_CIPHER(state, job); + else + job = CALL_SUBMIT_HASH(state, job); + + job = RESUBMIT_BURST_JOB(state, job); + return job; +} + +__forceinline +uint32_t complete_burst_job(IMB_MGR *state, IMB_JOB *job) +{ + uint32_t completed_jobs = 0; + + /** + * complete as many jobs as necessary + * until specified 'job' has completed + */ + if (job->chain_order == IMB_ORDER_CIPHER_HASH) { + /* while() loop optimized for cipher_hash order */ + while (job->status < IMB_STATUS_COMPLETED) { + IMB_JOB *tmp = CALL_FLUSH_CIPHER(state, job); + + if (tmp == NULL) + tmp = CALL_FLUSH_HASH(state, job); + + (void) RESUBMIT_BURST_JOB(state, tmp); + completed_jobs++; + } + } else { + /* while() loop optimized for hash_cipher order */ + while (job->status < IMB_STATUS_COMPLETED) { + IMB_JOB *tmp = CALL_FLUSH_HASH(state, job); + + if (tmp == NULL) + tmp = CALL_FLUSH_CIPHER(state, job); + + (void) RESUBMIT_BURST_JOB(state, tmp); + completed_jobs++; + } + } + + return completed_jobs; +} + /* ========================================================================= */ /* ========================================================================= */ diff --git a/lib/include/mb_mgr_job_check.h b/lib/include/mb_mgr_job_check.h index 32cda0fbe74b8d19e1b71c52c026b3871c3cafe1..343af13aeec3e5d137a83302e17086a2f4b56df0 100644 --- a/lib/include/mb_mgr_job_check.h +++ b/lib/include/mb_mgr_job_check.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -153,6 +153,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } switch (cipher_mode) { +#ifndef __aarch64__ case IMB_CIPHER_CBC: case IMB_CIPHER_CBCS_1_9: if (job->src == NULL) { @@ -246,10 +247,6 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); return 1; } - if (job->iv_len_in_bytes != UINT64_C(0)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } break; case IMB_CIPHER_CNTR: case IMB_CIPHER_CNTR_BITLEN: @@ -293,6 +290,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#endif /* __aarch64__ */ case IMB_CIPHER_NULL: /* * No checks required for this mode @@ -300,6 +298,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, * from source to destination */ break; +#ifndef __aarch64__ case IMB_CIPHER_DOCSIS_SEC_BPI: if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); @@ -666,6 +665,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, } } break; +#endif /* __aarch64__ */ case IMB_CIPHER_ZUC_EEA3: if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); @@ -737,6 +737,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#ifndef __aarch64__ case IMB_CIPHER_KASUMI_UEA1_BITLEN: if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); @@ -864,12 +865,14 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#endif /* __aarch64__ */ default: imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } switch (hash_alg) { +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: case IMB_AUTH_MD5: case IMB_AUTH_HMAC_SHA_224: @@ -938,8 +941,10 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#endif /* __aarch64__ */ case IMB_AUTH_NULL: break; +#ifndef __aarch64__ case IMB_AUTH_CRC32_ETHERNET_FCS: case IMB_AUTH_CRC32_SCTP: case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: @@ -1231,6 +1236,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#endif /* __aarch64__ */ case IMB_AUTH_ZUC_EIA3_BITLEN: if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); @@ -1291,6 +1297,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#ifndef __aarch64__ case IMB_AUTH_DOCSIS_CRC32: /** * Use only in combination with DOCSIS_SEC_BPI. @@ -1350,6 +1357,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#endif /* __aarch64__ */ case IMB_AUTH_SNOW3G_UIA2_BITLEN: if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); @@ -1378,6 +1386,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#ifndef __aarch64__ case IMB_AUTH_KASUMI_UIA1: if (job->src == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); @@ -1507,6 +1516,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, return 1; } break; +#endif /* __aarch64__ */ default: imb_set_errno(state, IMB_ERR_HASH_ALGO); return 1; diff --git a/lib/include/memcpy.h b/lib/include/memcpy.h index 20e9ac353b4f7be6a41a701de18fd739aa1c71e5..89fa0ded5ef17547af9fa62c58f849761b40cf59 100644 --- a/lib/include/memcpy.h +++ b/lib/include/memcpy.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/memcpy.asm b/lib/include/memcpy.inc similarity index 99% rename from lib/include/memcpy.asm rename to lib/include/memcpy.inc index ad85b20684ef26849f4a8b77b7d7d8a8ef64bc60..2596b2b6d8ad701641ccbbb7975387b975d3c287 100644 --- a/lib/include/memcpy.asm +++ b/lib/include/memcpy.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef __MEMCPY_ASM__ -%define __MEMCPY_ASM__ +%ifndef __MEMCPY_INC__ +%define __MEMCPY_INC__ -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" ; This section defines a series of macros to copy small to medium amounts ; of data from memory to memory, where the size is variable but limited. @@ -766,4 +766,4 @@ %%end: %endm -%endif ; ifndef __MEMCPY_ASM__ +%endif ; ifndef __MEMCPY_INC__ diff --git a/lib/include/noaesni.h b/lib/include/noaesni.h index be68ad85ac104e24e346aeacc5cb4c1abf8a77c5..f97a1e85ad1aa678df9cbc6cce654373055ea5ca 100644 --- a/lib/include/noaesni.h +++ b/lib/include/noaesni.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -30,6 +30,7 @@ #ifndef NOAESNI_H #define NOAESNI_H +#ifdef __aarch64__ IMB_DLL_EXPORT void init_mb_mgr_aarch64_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *submit_job_aarch64_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_aarch64_no_aesni(IMB_MGR *state); @@ -37,6 +38,7 @@ IMB_DLL_EXPORT IMB_JOB *flush_job_aarch64_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT uint32_t queue_size_aarch64_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_aarch64_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_aarch64_no_aesni(IMB_MGR *state); +#else /* __aarch64__ */ IMB_DLL_EXPORT void init_mb_mgr_sse_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *submit_job_sse_no_aesni(IMB_MGR *state); @@ -106,4 +108,9 @@ IMB_DLL_EXPORT void aes_cfb_128_one_sse_no_aesni(void *out, const void *in, const void *iv, const void *keys, uint64_t len); +IMB_DLL_EXPORT void +aes_cfb_256_one_sse_no_aesni(void *out, const void *in, const void *iv, + const void *keys, uint64_t len); +#endif /* __aarch64__ */ + #endif /* NOAESNI_H */ diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index 1d4c94761d2dfec67cf72c19b9bfe098a14a9eb5..270b40d8d1000f93068ab19fd044b43bc92c42b1 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -28,6 +28,7 @@ #ifndef OOO_MGR_RESET_H #define OOO_MGR_RESET_H +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes); @@ -60,10 +61,12 @@ void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes); +#endif /* __aarch64__ */ IMB_DLL_LOCAL void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes); +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); @@ -75,6 +78,7 @@ void ooo_mgr_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes); +#endif /* __aarch64__ */ IMB_DLL_LOCAL void ooo_mgr_snow3g_reset(void *p_ooo_mgr, const unsigned num_lanes); diff --git a/lib/include/os.asm b/lib/include/os.inc similarity index 96% rename from lib/include/os.asm rename to lib/include/os.inc index 90d1c5ab6491393f4e37baa1566ee9e4f21bc116..f0f3092071a6bf567aea3daf7529e08ddd5a0178 100644 --- a/lib/include/os.asm +++ b/lib/include/os.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -24,8 +24,8 @@ ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef OS_ASM_FILE -%define OS_ASM_FILE +%ifndef OS_INC_FILE +%define OS_INC_FILE %ifndef WIN_ABI %ifidn __OUTPUT_FORMAT__, win64 @@ -114,4 +114,4 @@ %endif ; LINUX %endmacro -%endif ; OS_ASM_FILE +%endif ; OS_INC_FILE diff --git a/lib/include/reg_sizes.asm b/lib/include/reg_sizes.asm deleted file mode 100644 index c2845e9049a1edebefee33ad68b2215ec174f8e4..0000000000000000000000000000000000000000 --- a/lib/include/reg_sizes.asm +++ /dev/null @@ -1,300 +0,0 @@ -;; -;; Copyright (c) 2012-2022, Intel Corporation -;; -;; Redistribution and use in source and binary forms, with or without -;; modification, are permitted provided that the following conditions are met: -;; -;; * Redistributions of source code must retain the above copyright notice, -;; this list of conditions and the following disclaimer. -;; * Redistributions in binary form must reproduce the above copyright -;; notice, this list of conditions and the following disclaimer in the -;; documentation and/or other materials provided with the distribution. -;; * Neither the name of Intel Corporation nor the names of its contributors -;; may be used to endorse or promote products derived from this software -;; without specific prior written permission. -;; -;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;; - -; define d and w variants for registers - -%ifndef _REG_SIZES_ASM_ -%define _REG_SIZES_ASM_ - -%define raxd eax -%define raxw ax -%define raxb al - -%define rbxd ebx -%define rbxw bx -%define rbxb bl - -%define rcxd ecx -%define rcxw cx -%define rcxb cl - -%define rdxd edx -%define rdxw dx -%define rdxb dl - -%define rsid esi -%define rsiw si -%define rsib sil - -%define rdid edi -%define rdiw di -%define rdib dil - -%define rbpd ebp -%define rbpw bp -%define rbpb bpl - -%define zmm0x xmm0 -%define zmm1x xmm1 -%define zmm2x xmm2 -%define zmm3x xmm3 -%define zmm4x xmm4 -%define zmm5x xmm5 -%define zmm6x xmm6 -%define zmm7x xmm7 -%define zmm8x xmm8 -%define zmm9x xmm9 -%define zmm10x xmm10 -%define zmm11x xmm11 -%define zmm12x xmm12 -%define zmm13x xmm13 -%define zmm14x xmm14 -%define zmm15x xmm15 -%define zmm16x xmm16 -%define zmm17x xmm17 -%define zmm18x xmm18 -%define zmm19x xmm19 -%define zmm20x xmm20 -%define zmm21x xmm21 -%define zmm22x xmm22 -%define zmm23x xmm23 -%define zmm24x xmm24 -%define zmm25x xmm25 -%define zmm26x xmm26 -%define zmm27x xmm27 -%define zmm28x xmm28 -%define zmm29x xmm29 -%define zmm30x xmm30 -%define zmm31x xmm31 - -%define ymm0x xmm0 -%define ymm1x xmm1 -%define ymm2x xmm2 -%define ymm3x xmm3 -%define ymm4x xmm4 -%define ymm5x xmm5 -%define ymm6x xmm6 -%define ymm7x xmm7 -%define ymm8x xmm8 -%define ymm9x xmm9 -%define ymm10x xmm10 -%define ymm11x xmm11 -%define ymm12x xmm12 -%define ymm13x xmm13 -%define ymm14x xmm14 -%define ymm15x xmm15 -%define ymm16x xmm16 -%define ymm17x xmm17 -%define ymm18x xmm18 -%define ymm19x xmm19 -%define ymm20x xmm20 -%define ymm21x xmm21 -%define ymm22x xmm22 -%define ymm23x xmm23 -%define ymm24x xmm24 -%define ymm25x xmm25 -%define ymm26x xmm26 -%define ymm27x xmm27 -%define ymm28x xmm28 -%define ymm29x xmm29 -%define ymm30x xmm30 -%define ymm31x xmm31 - -%define xmm0x xmm0 -%define xmm1x xmm1 -%define xmm2x xmm2 -%define xmm3x xmm3 -%define xmm4x xmm4 -%define xmm5x xmm5 -%define xmm6x xmm6 -%define xmm7x xmm7 -%define xmm8x xmm8 -%define xmm9x xmm9 -%define xmm10x xmm10 -%define xmm11x xmm11 -%define xmm12x xmm12 -%define xmm13x xmm13 -%define xmm14x xmm14 -%define xmm15x xmm15 -%define xmm16x xmm16 -%define xmm17x xmm17 -%define xmm18x xmm18 -%define xmm19x xmm19 -%define xmm20x xmm20 -%define xmm21x xmm21 -%define xmm22x xmm22 -%define xmm23x xmm23 -%define xmm24x xmm24 -%define xmm25x xmm25 -%define xmm26x xmm26 -%define xmm27x xmm27 -%define xmm28x xmm28 -%define xmm29x xmm29 -%define xmm30x xmm30 -%define xmm31x xmm31 - -%define zmm0y ymm0 -%define zmm1y ymm1 -%define zmm2y ymm2 -%define zmm3y ymm3 -%define zmm4y ymm4 -%define zmm5y ymm5 -%define zmm6y ymm6 -%define zmm7y ymm7 -%define zmm8y ymm8 -%define zmm9y ymm9 -%define zmm10y ymm10 -%define zmm11y ymm11 -%define zmm12y ymm12 -%define zmm13y ymm13 -%define zmm14y ymm14 -%define zmm15y ymm15 -%define zmm16y ymm16 -%define zmm17y ymm17 -%define zmm18y ymm18 -%define zmm19y ymm19 -%define zmm20y ymm20 -%define zmm21y ymm21 -%define zmm22y ymm22 -%define zmm23y ymm23 -%define zmm24y ymm24 -%define zmm25y ymm25 -%define zmm26y ymm26 -%define zmm27y ymm27 -%define zmm28y ymm28 -%define zmm29y ymm29 -%define zmm30y ymm30 -%define zmm31y ymm31 - -%define xmm0y ymm0 -%define xmm1y ymm1 -%define xmm2y ymm2 -%define xmm3y ymm3 -%define xmm4y ymm4 -%define xmm5y ymm5 -%define xmm6y ymm6 -%define xmm7y ymm7 -%define xmm8y ymm8 -%define xmm9y ymm9 -%define xmm10y ymm10 -%define xmm11y ymm11 -%define xmm12y ymm12 -%define xmm13y ymm13 -%define xmm14y ymm14 -%define xmm15y ymm15 -%define xmm16y ymm16 -%define xmm17y ymm17 -%define xmm18y ymm18 -%define xmm19y ymm19 -%define xmm20y ymm20 -%define xmm21y ymm21 -%define xmm22y ymm22 -%define xmm23y ymm23 -%define xmm24y ymm24 -%define xmm25y ymm25 -%define xmm26y ymm26 -%define xmm27y ymm27 -%define xmm28y ymm28 -%define xmm29y ymm29 -%define xmm30y ymm30 -%define xmm31y ymm31 - -%define xmm0z zmm0 -%define xmm1z zmm1 -%define xmm2z zmm2 -%define xmm3z zmm3 -%define xmm4z zmm4 -%define xmm5z zmm5 -%define xmm6z zmm6 -%define xmm7z zmm7 -%define xmm8z zmm8 -%define xmm9z zmm9 -%define xmm10z zmm10 -%define xmm11z zmm11 -%define xmm12z zmm12 -%define xmm13z zmm13 -%define xmm14z zmm14 -%define xmm15z zmm15 -%define xmm16z zmm16 -%define xmm17z zmm17 -%define xmm18z zmm18 -%define xmm19z zmm19 -%define xmm20z zmm20 -%define xmm21z zmm21 -%define xmm22z zmm22 -%define xmm23z zmm23 -%define xmm24z zmm24 -%define xmm25z zmm25 -%define xmm26z zmm26 -%define xmm27z zmm27 -%define xmm28z zmm28 -%define xmm29z zmm29 -%define xmm30z zmm30 -%define xmm31z zmm31 - -%define ymm0z zmm0 -%define ymm1z zmm1 -%define ymm2z zmm2 -%define ymm3z zmm3 -%define ymm4z zmm4 -%define ymm5z zmm5 -%define ymm6z zmm6 -%define ymm7z zmm7 -%define ymm8z zmm8 -%define ymm9z zmm9 -%define ymm10z zmm10 -%define ymm11z zmm11 -%define ymm12z zmm12 -%define ymm13z zmm13 -%define ymm14z zmm14 -%define ymm15z zmm15 -%define ymm16z zmm16 -%define ymm17z zmm17 -%define ymm18z zmm18 -%define ymm19z zmm19 -%define ymm20z zmm20 -%define ymm21z zmm21 -%define ymm22z zmm22 -%define ymm23z zmm23 -%define ymm24z zmm24 -%define ymm25z zmm25 -%define ymm26z zmm26 -%define ymm27z zmm27 -%define ymm28z zmm28 -%define ymm29z zmm29 -%define ymm30z zmm30 -%define ymm31z zmm31 - -%define DWORD(reg) reg %+ d -%define WORD(reg) reg %+ w -%define BYTE(reg) reg %+ b - -%define XWORD(reg) reg %+ x -%define YWORD(reg) reg %+ y -%define ZWORD(reg) reg %+ z - -%endif ;; _REG_SIZES_ASM_ diff --git a/lib/include/reg_sizes.inc b/lib/include/reg_sizes.inc new file mode 100644 index 0000000000000000000000000000000000000000..42f14e2db84917d65f4130660f6a7bac564d495c --- /dev/null +++ b/lib/include/reg_sizes.inc @@ -0,0 +1,366 @@ +;; +;; Copyright (c) 2012-2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; define d and w variants for registers + +%ifndef _REG_SIZES_INC_ +%define _REG_SIZES_INC_ + +%define raxd eax +%define raxw ax +%define raxb al + +%define rbxd ebx +%define rbxw bx +%define rbxb bl + +%define rcxd ecx +%define rcxw cx +%define rcxb cl + +%define rdxd edx +%define rdxw dx +%define rdxb dl + +%define rsid esi +%define rsiw si +%define rsib sil + +%define rdid edi +%define rdiw di +%define rdib dil + +%define rbpd ebp +%define rbpw bp +%define rbpb bpl + +%xdefine zmm0x xmm0 +%xdefine zmm1x xmm1 +%xdefine zmm2x xmm2 +%xdefine zmm3x xmm3 +%xdefine zmm4x xmm4 +%xdefine zmm5x xmm5 +%xdefine zmm6x xmm6 +%xdefine zmm7x xmm7 +%xdefine zmm8x xmm8 +%xdefine zmm9x xmm9 +%xdefine zmm10x xmm10 +%xdefine zmm11x xmm11 +%xdefine zmm12x xmm12 +%xdefine zmm13x xmm13 +%xdefine zmm14x xmm14 +%xdefine zmm15x xmm15 +%xdefine zmm16x xmm16 +%xdefine zmm17x xmm17 +%xdefine zmm18x xmm18 +%xdefine zmm19x xmm19 +%xdefine zmm20x xmm20 +%xdefine zmm21x xmm21 +%xdefine zmm22x xmm22 +%xdefine zmm23x xmm23 +%xdefine zmm24x xmm24 +%xdefine zmm25x xmm25 +%xdefine zmm26x xmm26 +%xdefine zmm27x xmm27 +%xdefine zmm28x xmm28 +%xdefine zmm29x xmm29 +%xdefine zmm30x xmm30 +%xdefine zmm31x xmm31 + +%xdefine ymm0x xmm0 +%xdefine ymm1x xmm1 +%xdefine ymm2x xmm2 +%xdefine ymm3x xmm3 +%xdefine ymm4x xmm4 +%xdefine ymm5x xmm5 +%xdefine ymm6x xmm6 +%xdefine ymm7x xmm7 +%xdefine ymm8x xmm8 +%xdefine ymm9x xmm9 +%xdefine ymm10x xmm10 +%xdefine ymm11x xmm11 +%xdefine ymm12x xmm12 +%xdefine ymm13x xmm13 +%xdefine ymm14x xmm14 +%xdefine ymm15x xmm15 +%xdefine ymm16x xmm16 +%xdefine ymm17x xmm17 +%xdefine ymm18x xmm18 +%xdefine ymm19x xmm19 +%xdefine ymm20x xmm20 +%xdefine ymm21x xmm21 +%xdefine ymm22x xmm22 +%xdefine ymm23x xmm23 +%xdefine ymm24x xmm24 +%xdefine ymm25x xmm25 +%xdefine ymm26x xmm26 +%xdefine ymm27x xmm27 +%xdefine ymm28x xmm28 +%xdefine ymm29x xmm29 +%xdefine ymm30x xmm30 +%xdefine ymm31x xmm31 + +%xdefine xmm0x xmm0 +%xdefine xmm1x xmm1 +%xdefine xmm2x xmm2 +%xdefine xmm3x xmm3 +%xdefine xmm4x xmm4 +%xdefine xmm5x xmm5 +%xdefine xmm6x xmm6 +%xdefine xmm7x xmm7 +%xdefine xmm8x xmm8 +%xdefine xmm9x xmm9 +%xdefine xmm10x xmm10 +%xdefine xmm11x xmm11 +%xdefine xmm12x xmm12 +%xdefine xmm13x xmm13 +%xdefine xmm14x xmm14 +%xdefine xmm15x xmm15 +%xdefine xmm16x xmm16 +%xdefine xmm17x xmm17 +%xdefine xmm18x xmm18 +%xdefine xmm19x xmm19 +%xdefine xmm20x xmm20 +%xdefine xmm21x xmm21 +%xdefine xmm22x xmm22 +%xdefine xmm23x xmm23 +%xdefine xmm24x xmm24 +%xdefine xmm25x xmm25 +%xdefine xmm26x xmm26 +%xdefine xmm27x xmm27 +%xdefine xmm28x xmm28 +%xdefine xmm29x xmm29 +%xdefine xmm30x xmm30 +%xdefine xmm31x xmm31 + +%xdefine zmm0y ymm0 +%xdefine zmm1y ymm1 +%xdefine zmm2y ymm2 +%xdefine zmm3y ymm3 +%xdefine zmm4y ymm4 +%xdefine zmm5y ymm5 +%xdefine zmm6y ymm6 +%xdefine zmm7y ymm7 +%xdefine zmm8y ymm8 +%xdefine zmm9y ymm9 +%xdefine zmm10y ymm10 +%xdefine zmm11y ymm11 +%xdefine zmm12y ymm12 +%xdefine zmm13y ymm13 +%xdefine zmm14y ymm14 +%xdefine zmm15y ymm15 +%xdefine zmm16y ymm16 +%xdefine zmm17y ymm17 +%xdefine zmm18y ymm18 +%xdefine zmm19y ymm19 +%xdefine zmm20y ymm20 +%xdefine zmm21y ymm21 +%xdefine zmm22y ymm22 +%xdefine zmm23y ymm23 +%xdefine zmm24y ymm24 +%xdefine zmm25y ymm25 +%xdefine zmm26y ymm26 +%xdefine zmm27y ymm27 +%xdefine zmm28y ymm28 +%xdefine zmm29y ymm29 +%xdefine zmm30y ymm30 +%xdefine zmm31y ymm31 + +%xdefine ymm0y ymm0 +%xdefine ymm1y ymm1 +%xdefine ymm2y ymm2 +%xdefine ymm3y ymm3 +%xdefine ymm4y ymm4 +%xdefine ymm5y ymm5 +%xdefine ymm6y ymm6 +%xdefine ymm7y ymm7 +%xdefine ymm8y ymm8 +%xdefine ymm9y ymm9 +%xdefine ymm10y ymm10 +%xdefine ymm11y ymm11 +%xdefine ymm12y ymm12 +%xdefine ymm13y ymm13 +%xdefine ymm14y ymm14 +%xdefine ymm15y ymm15 +%xdefine ymm16y ymm16 +%xdefine ymm17y ymm17 +%xdefine ymm18y ymm18 +%xdefine ymm19y ymm19 +%xdefine ymm20y ymm20 +%xdefine ymm21y ymm21 +%xdefine ymm22y ymm22 +%xdefine ymm23y ymm23 +%xdefine ymm24y ymm24 +%xdefine ymm25y ymm25 +%xdefine ymm26y ymm26 +%xdefine ymm27y ymm27 +%xdefine ymm28y ymm28 +%xdefine ymm29y ymm29 +%xdefine ymm30y ymm30 +%xdefine ymm31y ymm31 + +%xdefine xmm0y ymm0 +%xdefine xmm1y ymm1 +%xdefine xmm2y ymm2 +%xdefine xmm3y ymm3 +%xdefine xmm4y ymm4 +%xdefine xmm5y ymm5 +%xdefine xmm6y ymm6 +%xdefine xmm7y ymm7 +%xdefine xmm8y ymm8 +%xdefine xmm9y ymm9 +%xdefine xmm10y ymm10 +%xdefine xmm11y ymm11 +%xdefine xmm12y ymm12 +%xdefine xmm13y ymm13 +%xdefine xmm14y ymm14 +%xdefine xmm15y ymm15 +%xdefine xmm16y ymm16 +%xdefine xmm17y ymm17 +%xdefine xmm18y ymm18 +%xdefine xmm19y ymm19 +%xdefine xmm20y ymm20 +%xdefine xmm21y ymm21 +%xdefine xmm22y ymm22 +%xdefine xmm23y ymm23 +%xdefine xmm24y ymm24 +%xdefine xmm25y ymm25 +%xdefine xmm26y ymm26 +%xdefine xmm27y ymm27 +%xdefine xmm28y ymm28 +%xdefine xmm29y ymm29 +%xdefine xmm30y ymm30 +%xdefine xmm31y ymm31 + +%xdefine xmm0z zmm0 +%xdefine xmm1z zmm1 +%xdefine xmm2z zmm2 +%xdefine xmm3z zmm3 +%xdefine xmm4z zmm4 +%xdefine xmm5z zmm5 +%xdefine xmm6z zmm6 +%xdefine xmm7z zmm7 +%xdefine xmm8z zmm8 +%xdefine xmm9z zmm9 +%xdefine xmm10z zmm10 +%xdefine xmm11z zmm11 +%xdefine xmm12z zmm12 +%xdefine xmm13z zmm13 +%xdefine xmm14z zmm14 +%xdefine xmm15z zmm15 +%xdefine xmm16z zmm16 +%xdefine xmm17z zmm17 +%xdefine xmm18z zmm18 +%xdefine xmm19z zmm19 +%xdefine xmm20z zmm20 +%xdefine xmm21z zmm21 +%xdefine xmm22z zmm22 +%xdefine xmm23z zmm23 +%xdefine xmm24z zmm24 +%xdefine xmm25z zmm25 +%xdefine xmm26z zmm26 +%xdefine xmm27z zmm27 +%xdefine xmm28z zmm28 +%xdefine xmm29z zmm29 +%xdefine xmm30z zmm30 +%xdefine xmm31z zmm31 + +%xdefine ymm0z zmm0 +%xdefine ymm1z zmm1 +%xdefine ymm2z zmm2 +%xdefine ymm3z zmm3 +%xdefine ymm4z zmm4 +%xdefine ymm5z zmm5 +%xdefine ymm6z zmm6 +%xdefine ymm7z zmm7 +%xdefine ymm8z zmm8 +%xdefine ymm9z zmm9 +%xdefine ymm10z zmm10 +%xdefine ymm11z zmm11 +%xdefine ymm12z zmm12 +%xdefine ymm13z zmm13 +%xdefine ymm14z zmm14 +%xdefine ymm15z zmm15 +%xdefine ymm16z zmm16 +%xdefine ymm17z zmm17 +%xdefine ymm18z zmm18 +%xdefine ymm19z zmm19 +%xdefine ymm20z zmm20 +%xdefine ymm21z zmm21 +%xdefine ymm22z zmm22 +%xdefine ymm23z zmm23 +%xdefine ymm24z zmm24 +%xdefine ymm25z zmm25 +%xdefine ymm26z zmm26 +%xdefine ymm27z zmm27 +%xdefine ymm28z zmm28 +%xdefine ymm29z zmm29 +%xdefine ymm30z zmm30 +%xdefine ymm31z zmm31 + +%xdefine zmm0z zmm0 +%xdefine zmm1z zmm1 +%xdefine zmm2z zmm2 +%xdefine zmm3z zmm3 +%xdefine zmm4z zmm4 +%xdefine zmm5z zmm5 +%xdefine zmm6z zmm6 +%xdefine zmm7z zmm7 +%xdefine zmm8z zmm8 +%xdefine zmm9z zmm9 +%xdefine zmm10z zmm10 +%xdefine zmm11z zmm11 +%xdefine zmm12z zmm12 +%xdefine zmm13z zmm13 +%xdefine zmm14z zmm14 +%xdefine zmm15z zmm15 +%xdefine zmm16z zmm16 +%xdefine zmm17z zmm17 +%xdefine zmm18z zmm18 +%xdefine zmm19z zmm19 +%xdefine zmm20z zmm20 +%xdefine zmm21z zmm21 +%xdefine zmm22z zmm22 +%xdefine zmm23z zmm23 +%xdefine zmm24z zmm24 +%xdefine zmm25z zmm25 +%xdefine zmm26z zmm26 +%xdefine zmm27z zmm27 +%xdefine zmm28z zmm28 +%xdefine zmm29z zmm29 +%xdefine zmm30z zmm30 +%xdefine zmm31z zmm31 + +%define DWORD(reg) reg %+ d +%define WORD(reg) reg %+ w +%define BYTE(reg) reg %+ b + +%define XWORD(reg) reg %+ x +%define YWORD(reg) reg %+ y +%define ZWORD(reg) reg %+ z + +%endif ;; _REG_SIZES_INC_ diff --git a/lib/include/save_xmms.h b/lib/include/save_xmms.h index 88983c55aca2096f94e977aac2a645e911453198..1513c232705026c991d322fea4f99da526c77df1 100644 --- a/lib/include/save_xmms.h +++ b/lib/include/save_xmms.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/sha_generic.h b/lib/include/sha_generic.h index 9329da22d21ca5f22ab5dc1e665b03acf832c7e1..c7f51e72db449a3fb1f8c2fee0388f1643509aaf 100644 --- a/lib/include/sha_generic.h +++ b/lib/include/sha_generic.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -39,12 +39,14 @@ extern void sha1_block_sse(const void *, void *); extern void sha1_block_avx(const void *, void *); +extern void sha1_ni_block_sse(const void *, void *); extern void sha224_block_sse(const void *, void *); extern void sha224_block_avx(const void *, void *); extern void sha256_block_sse(const void *, void *); extern void sha256_block_avx(const void *, void *); +extern void sha256_ni_block_sse(const void *, void *); extern void sha384_block_sse(const void *, void *); extern void sha384_block_avx(const void *, void *); @@ -52,6 +54,11 @@ extern void sha384_block_avx(const void *, void *); extern void sha512_block_sse(const void *, void *); extern void sha512_block_avx(const void *, void *); +enum arch_type { + ARCH_SSE = 0, + ARCH_SSE_SHANI, + ARCH_AVX +}; /* ========================================================================== */ /* @@ -116,30 +123,37 @@ void copy_bswap8_array(void *dst, const void *src, const size_t num) __forceinline void sha_generic_one_block(const void *inp, void *digest, - const int is_avx, const int sha_type) + const enum arch_type arch, const int sha_type) { if (sha_type == 1) { - if (is_avx) + if (arch == ARCH_AVX) sha1_block_avx(inp, digest); - else + else if (arch == ARCH_SSE) sha1_block_sse(inp, digest); + else /* arch == ARCH_SSE_SHANI */ + sha1_ni_block_sse(inp, digest); } else if (sha_type == 224) { - if (is_avx) + if (arch == ARCH_AVX) sha224_block_avx(inp, digest); - else + else if (arch == ARCH_SSE) sha224_block_sse(inp, digest); + else /* arch == ARCH_SSE_SHANI */ + /* Same as SHA-224 */ + sha256_ni_block_sse(inp, digest); } else if (sha_type == 256) { - if (is_avx) + if (arch == ARCH_AVX) sha256_block_avx(inp, digest); - else + else if (arch == ARCH_SSE) sha256_block_sse(inp, digest); + else /* arch == ARCH_SSE_SHANI */ + sha256_ni_block_sse(inp, digest); } else if (sha_type == 384) { - if (is_avx) + if (arch == ARCH_AVX) sha384_block_avx(inp, digest); else sha384_block_sse(inp, digest); } else if (sha_type == 512) { - if (is_avx) + if (arch == ARCH_AVX) sha512_block_avx(inp, digest); else sha512_block_sse(inp, digest); @@ -252,8 +266,8 @@ void sha_generic_write_digest(void *dst, const void *src, const int sha_type) __forceinline void sha_generic(const void *data, const uint64_t length, void *digest, - const int is_avx, const int sha_type, const uint64_t blk_size, - const uint64_t pad_size) + const enum arch_type arch, const int sha_type, + const uint64_t blk_size, const uint64_t pad_size) { #ifdef SAFE_PARAM imb_set_errno(NULL, 0); @@ -279,7 +293,7 @@ sha_generic(const void *data, const uint64_t length, void *digest, sha_generic_init(ld, sha_type); for (idx = 0; (idx + blk_size) <= length; idx += blk_size) - sha_generic_one_block(&inp[idx], ld, is_avx, sha_type); + sha_generic_one_block(&inp[idx], ld, arch, sha_type); r = length % blk_size; @@ -289,19 +303,19 @@ sha_generic(const void *data, const uint64_t length, void *digest, if (r >= (blk_size - pad_size)) { /* length will be encoded in the next block */ - sha_generic_one_block(cb, ld, is_avx, sha_type); + sha_generic_one_block(cb, ld, arch, sha_type); memset(cb, 0, sizeof(cb)); } store8_be(&cb[blk_size - 8], length * 8 /* bit length */); - sha_generic_one_block(cb, ld, is_avx, sha_type); + sha_generic_one_block(cb, ld, arch, sha_type); sha_generic_write_digest(digest, ld, sha_type); #ifdef SAFE_DATA clear_mem(cb, sizeof(cb)); clear_mem(&local_digest, sizeof(local_digest)); clear_scratch_gps(); - if (is_avx) + if (arch == ARCH_AVX) clear_scratch_xmms_avx(); else clear_scratch_xmms_sse(); @@ -310,7 +324,7 @@ sha_generic(const void *data, const uint64_t length, void *digest, __forceinline void sha_generic_1block(const void *data, void *digest, - const int is_avx, const int sha_type) + const enum arch_type arch, const int sha_type) { #ifdef SAFE_PARAM imb_set_errno(NULL, 0); @@ -324,10 +338,10 @@ void sha_generic_1block(const void *data, void *digest, } #endif sha_generic_init(digest, sha_type); - sha_generic_one_block(data, digest, is_avx, sha_type); + sha_generic_one_block(data, digest, arch, sha_type); #ifdef SAFE_DATA clear_scratch_gps(); - if (is_avx) + if (arch == ARCH_AVX) clear_scratch_xmms_avx(); else clear_scratch_xmms_sse(); diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 8b4ce7216797e37ab1f1baf369c696686dd22137..c2e61e334bc5d8aed4398c044a4f7d02782d92a2 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/snow3g.h b/lib/include/snow3g.h index 12326b0b2ebf00d54a335bcf58473fc699ee7835..cd7eca39fcc7f40a1e9563829a5b95423d80a1da 100644 --- a/lib/include/snow3g.h +++ b/lib/include/snow3g.h @@ -28,6 +28,7 @@ #ifndef _SNOW3G_H_ #define _SNOW3G_H_ +#ifndef __aarch64__ /******************************************************************************* * SSE ******************************************************************************/ @@ -655,6 +656,7 @@ snow3g_f9_1_buffer_internal_vaes_avx512(const uint64_t *pBufferIn, const uint32_t KS[5], const uint64_t lengthInBits); +#else /* __aarch64__ */ /***************************************************************************** * AARCH64 ******************************************************************************/ @@ -1093,4 +1095,6 @@ int snow3g_init_key_sched_aarch64_no_aesni(const void *pKey, snow3g_key_schedule_t *pCtx); +#endif /* __aarch64__ */ + #endif /* _SNOW3G_H_ */ diff --git a/lib/include/snow3g_common.h b/lib/include/snow3g_common.h index 75fd952cd4497d24bdf5e5b34740e8fa6247f1d8..8d9917fd3999175936b24ce5cb61312f8d22fa4d 100644 --- a/lib/include/snow3g_common.h +++ b/lib/include/snow3g_common.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -2008,66 +2008,116 @@ snow3gStateInitialize_8_multiKey(snow3gKeyState8_t *pCtx, const snow3g_key_schedule_t * const KeySched[], const void * const pIV[]) { - DECLARE_ALIGNED(uint32_t k[8], 32); - DECLARE_ALIGNED(uint32_t l[8], 32); - __m256i *K = (__m256i *)k; - __m256i *L = (__m256i *)l; - - int i, j; - __m256i mR, mS, mT, mU, T0, T1; - - /* Initialize the LFSR table from constants, Keys, and IV */ - - /* Load complete 256b IV into register (SSE2)*/ static const __m256i swapMask = { 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL }; - mR = load_2xm128i_into_m256i(pIV[4], pIV[0]); - mS = load_2xm128i_into_m256i(pIV[5], pIV[1]); - mT = load_2xm128i_into_m256i(pIV[6], pIV[2]); - mU = load_2xm128i_into_m256i(pIV[7], pIV[3]); + static const __m256i ALLFS = { + 0xffffffffffffffffULL, 0xffffffffffffffffULL, + 0xffffffffffffffffULL, 0xffffffffffffffffULL + }; + __m256i ksR, ksS, ksT, ksU; + __m256i ivR, ivS, ivT, ivU; + __m256i T0, T1; + int i; - /* initialize the array block (SSE4) */ - for (i = 0; i < 4; i++) { - for (j = 0; j < 8; j++) { - k[j] = KeySched[j]->k[i]; - l[j] = ~k[j]; - } + /* + * Initialize the LFSR table from constants, Keys, and IV + * - First initialize LFSR with the key schedules + */ - pCtx->LFSR_X[i + 4] = *K; - pCtx->LFSR_X[i + 12] = *K; - pCtx->LFSR_X[i + 0] = *L; - pCtx->LFSR_X[i + 8] = *L; - } + /* Transform 8 x (4 x 32-bits) into 4 x (8 x 32-bits) */ - /* Update the schedule structure with IVs */ - /* Store the 4 IVs in LFSR by a column/row matrix swap - * after endianness correction */ + /* Load complete 256b keyschedule into registers */ + ksR = load_2xm128i_into_m256i(KeySched[4]->k, KeySched[0]->k); + ksS = load_2xm128i_into_m256i(KeySched[5]->k, KeySched[1]->k); + ksT = load_2xm128i_into_m256i(KeySched[6]->k, KeySched[2]->k); + ksU = load_2xm128i_into_m256i(KeySched[7]->k, KeySched[3]->k); - /* endianness swap */ - mR = _mm256_shuffle_epi8(mR, swapMask); - mS = _mm256_shuffle_epi8(mS, swapMask); - mT = _mm256_shuffle_epi8(mT, swapMask); - mU = _mm256_shuffle_epi8(mU, swapMask); + /* + * Ln -> lane index 'n' + * Wn -> 32-bit word index 'n' + * + * ksR = L4W3L4W2L4W1L4W0|L0W3L0W2L0W1L0W0 + * ksS = L5W3L5W2L5W1L5W0|L1W3L1W2L1W1L1W0 + * ksT = L6W3L6W2L6W1L6W0|L2W3L2W2L2W1L2W0 + * ksU = L7W3L7W2L7W1L7W0|L3W3L3W2L3W1L3W0 + */ - /* row/column dword inversion */ - T0 = _mm256_unpacklo_epi32(mR, mS); - mR = _mm256_unpackhi_epi32(mR, mS); - T1 = _mm256_unpacklo_epi32(mT, mU); - mT = _mm256_unpackhi_epi32(mT, mU); + T0 = _mm256_unpacklo_epi32(ksR, ksS); + ksR = _mm256_unpackhi_epi32(ksR, ksS); + T1 = _mm256_unpacklo_epi32(ksT, ksU); + ksT = _mm256_unpackhi_epi32(ksT, ksU); - /* row/column qword inversion */ - mU = _mm256_unpackhi_epi64(mR, mT); - mT = _mm256_unpacklo_epi64(mR, mT); - mS = _mm256_unpackhi_epi64(T0, T1); - mR = _mm256_unpacklo_epi64(T0, T1); + /* + * T0 = L5W1L4W1L5W0L4W0|L1W1L0W1L1W0L0W0 + * ksR = L5W3L4W3L5W2L4W2|L1W3L0W3L1W2L0W2 + * T1 = L7W1L6W1L7W0L6W0|L3W1L2W1L3W0L2W0 + * ksT = L7W3L6W3L7W2L6W2|L3W3L2W3L3W2L2W2 + */ + + ksU = _mm256_unpackhi_epi64(ksR, ksT); + ksT = _mm256_unpacklo_epi64(ksR, ksT); + ksS = _mm256_unpackhi_epi64(T0, T1); + ksR = _mm256_unpacklo_epi64(T0, T1); + + /* + * ksU = L7W3L6W3L5W3L4W3|L3W3L2W3L1W3L0W3 + * ksT = L7W2L6W2L5W2L4W2|L3W2L2W2L1W2L0W2 + * ksS = L7W1L6W1L5W1L4W1|L3W1L2W1L1W1L0W1 + * ksR = L7W0L6W0L5W0L4W0|L3W0L2W0L1W0L0W0 + */ + + pCtx->LFSR_X[4 + 0] = ksR; /* LFSR[12] set later */ + pCtx->LFSR_X[4 + 1] = pCtx->LFSR_X[12 + 1] = ksS; + pCtx->LFSR_X[4 + 2] = pCtx->LFSR_X[12 + 2] = ksT; + pCtx->LFSR_X[4 + 3] = ksU; /* LFSR[15] set later */ + + /* keep ksR & ksU and ~ksS & ~ksT for later */ + ksS = _mm256_xor_si256(ksS, ALLFS); + ksT = _mm256_xor_si256(ksT, ALLFS); + + pCtx->LFSR_X[0 + 0] = + pCtx->LFSR_X[8 + 0] = _mm256_xor_si256(ksR, ALLFS); + pCtx->LFSR_X[0 + 1] = ksS; /* LFSR[9] set later */ + pCtx->LFSR_X[0 + 2] = ksT; /* LFSR[10] set later */ + pCtx->LFSR_X[0 + 3] = + pCtx->LFSR_X[8 + 3] = _mm256_xor_si256(ksU, ALLFS); + + /* + * Update LFSR structure with IVs + * - Same transform as above, 8 x (4 x 32-bits) into 4 x (8 x 32-bits) + */ + + /* Load complete 256b IV into register */ + ivR = load_2xm128i_into_m256i(pIV[4], pIV[0]); + ivS = load_2xm128i_into_m256i(pIV[5], pIV[1]); + ivT = load_2xm128i_into_m256i(pIV[6], pIV[2]); + ivU = load_2xm128i_into_m256i(pIV[7], pIV[3]); + + /* endianness swap of the double words */ + ivR = _mm256_shuffle_epi8(ivR, swapMask); + ivS = _mm256_shuffle_epi8(ivS, swapMask); + ivT = _mm256_shuffle_epi8(ivT, swapMask); + ivU = _mm256_shuffle_epi8(ivU, swapMask); + + /* transpose */ + T0 = _mm256_unpacklo_epi32(ivR, ivS); + ivR = _mm256_unpackhi_epi32(ivR, ivS); + T1 = _mm256_unpacklo_epi32(ivT, ivU); + ivT = _mm256_unpackhi_epi32(ivT, ivU); + + ivU = _mm256_unpackhi_epi64(ivR, ivT); + ivT = _mm256_unpacklo_epi64(ivR, ivT); + ivS = _mm256_unpackhi_epi64(T0, T1); + ivR = _mm256_unpacklo_epi64(T0, T1); + + /* IV ^ LFSR */ + pCtx->LFSR_X[15] = _mm256_xor_si256(ksU, ivU); /* ksU ^ ivU */ + pCtx->LFSR_X[12] = _mm256_xor_si256(ksR, ivT); /* ksR ^ ivT */ + pCtx->LFSR_X[10] = _mm256_xor_si256(ksT, ivS); /* ~ksT ^ ivS */ + pCtx->LFSR_X[9] = _mm256_xor_si256(ksS, ivR); /* ~ksS ^ ivR */ - /*IV ^ LFSR */ - pCtx->LFSR_X[15] = _mm256_xor_si256(pCtx->LFSR_X[15], mU); - pCtx->LFSR_X[12] = _mm256_xor_si256(pCtx->LFSR_X[12], mT); - pCtx->LFSR_X[10] = _mm256_xor_si256(pCtx->LFSR_X[10], mS); - pCtx->LFSR_X[9] = _mm256_xor_si256(pCtx->LFSR_X[9], mR); pCtx->iLFSR_X = 0; /* FSM initialization */ @@ -2077,12 +2127,12 @@ snow3gStateInitialize_8_multiKey(snow3gKeyState8_t *pCtx, /* Initialisation rounds */ for (i = 0; i < 32; i++) { - mS = ClockFSM_8(pCtx); + T0 = ClockFSM_8(pCtx); ClockLFSR_8(pCtx); const uint32_t idx = (pCtx->iLFSR_X + 15) & 15; - pCtx->LFSR_X[idx] = _mm256_xor_si256(pCtx->LFSR_X[idx], mS); + pCtx->LFSR_X[idx] = _mm256_xor_si256(pCtx->LFSR_X[idx], T0); } } @@ -3261,12 +3311,12 @@ void SNOW3G_F8_N_BUFFER(const snow3g_key_schedule_t *pCtx, if (packetCount > NUM_PACKETS_16) { pBufferOut[0] = NULL; - printf("packetCount too high (%d)\n", packetCount); + printf("packetCount too high (%u)\n", (unsigned) packetCount); return; } uint32_t packet_index, inner_index, pktCnt = packetCount; - int sortNeeded = 0, tempLen = 0; + int sortNeeded = 0; uint8_t *srctempbuff; uint8_t *dsttempbuff; uint8_t *ivtempbuff; @@ -3316,7 +3366,9 @@ void SNOW3G_F8_N_BUFFER(const snow3g_key_schedule_t *pCtx, srctempbuff = pSrcBuf[packet_index]; dsttempbuff = pDstBuf[packet_index]; ivtempbuff = pIV[packet_index]; - tempLen = lensBuf[packet_index]; + + const int tempLen = + lensBuf[packet_index]; pSrcBuf[packet_index] = pSrcBuf[inner_index]; @@ -3465,12 +3517,12 @@ void SNOW3G_F8_N_BUFFER_MULTIKEY(const snow3g_key_schedule_t * const pCtx[], if (packetCount > NUM_PACKETS_16) { pBufferOut[0] = NULL; - printf("packetCount too high (%d)\n", packetCount); + printf("packetCount too high (%u)\n", (unsigned) packetCount); return; } uint32_t packet_index, inner_index, pktCnt = packetCount; - int sortNeeded = 0, tempLen = 0; + int sortNeeded = 0; uint8_t *srctempbuff; uint8_t *dsttempbuff; uint8_t *ivtempbuff; @@ -3519,7 +3571,10 @@ void SNOW3G_F8_N_BUFFER_MULTIKEY(const snow3g_key_schedule_t * const pCtx[], srctempbuff = pSrcBuf[packet_index]; dsttempbuff = pDstBuf[packet_index]; ivtempbuff = pIV[packet_index]; - tempLen = lensBuf[packet_index]; + + const int tempLen = + lensBuf[packet_index]; + tempCtx = pCtxBuf[packet_index]; pSrcBuf[packet_index] = diff --git a/lib/include/snow3g_submit.h b/lib/include/snow3g_submit.h index 72872220a79a2b2faaf6343ef94b75353514000b..9d98ddee3d18e6d6c6207332e9c36eb78d8d2308 100644 --- a/lib/include/snow3g_submit.h +++ b/lib/include/snow3g_submit.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/include/snow3g_tables.h b/lib/include/snow3g_tables.h index bc87e6da91d899cb89854b05801e183017c3b90f..f3cbf2189cfef732d60e683b383dde8f074003dd 100644 --- a/lib/include/snow3g_tables.h +++ b/lib/include/snow3g_tables.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -31,7 +31,7 @@ #include #include "ipsec-mb.h" -#ifdef __x86_64__ +#ifndef __aarch64__ #include "constant_lookup.h" #if defined (AVX) || defined (AVX2) @@ -40,7 +40,7 @@ #define SNOW3G_SAFE_LUT8(table, idx, size) LOOKUP8_SSE(table, idx, size) #endif /* AVX || AVX2 */ -#endif /* __x86_64__ */ +#endif /* __aarch64__ */ extern const int snow3g_table_A_mul[256]; extern const int snow3g_table_A_div[256]; diff --git a/lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm b/lib/include/snow3g_uea2_by16_vaes_avx512.inc similarity index 99% rename from lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm rename to lib/include/snow3g_uea2_by16_vaes_avx512.inc index d295d9abaa2cc942a06e5726f3f8a873df520f13..6eea8072c9b0ed946c7946172a0ac6bb411fbfd4 100644 --- a/lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm +++ b/lib/include/snow3g_uea2_by16_vaes_avx512.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -24,10 +24,10 @@ ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/mb_mgr_datastruct.asm" -%include "include/transpose_avx512.asm" -%include "include/imb_job.asm" -%include "include/constant_lookup.asm" +%include "include/mb_mgr_datastruct.inc" +%include "include/transpose_avx512.inc" +%include "include/imb_job.inc" +%include "include/constant_lookup.inc" mksection .rodata default rel diff --git a/lib/sse_t1/snow3g_uea2_by4_sse.asm b/lib/include/snow3g_uea2_by4_sse.inc similarity index 99% rename from lib/sse_t1/snow3g_uea2_by4_sse.asm rename to lib/include/snow3g_uea2_by4_sse.inc index 5b3a3561b57abcfcb6d33bf9c149f7a2001f696c..5542177643b8923c5f2b46483122a7e2eb5be0db 100644 --- a/lib/sse_t1/snow3g_uea2_by4_sse.asm +++ b/lib/include/snow3g_uea2_by4_sse.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,14 +25,14 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" -%include "include/imb_job.asm" -%include "include/clear_regs.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/memcpy.asm" -%include "include/transpose_sse.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/memcpy.inc" +%include "include/transpose_sse.inc" extern snow3g_table_A_mul extern snow3g_table_A_div diff --git a/lib/include/transpose_avx2.asm b/lib/include/transpose_avx2.inc similarity index 98% rename from lib/include/transpose_avx2.asm rename to lib/include/transpose_avx2.inc index 56e18a78e23d1ca124b1c3861918210dd71de771..4469a838a584fb9566ac4add901a62cc545499a1 100644 --- a/lib/include/transpose_avx2.asm +++ b/lib/include/transpose_avx2.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef _TRANSPOSE_AVX2_ASM_ -%define _TRANSPOSE_AVX2_ASM_ +%ifndef _TRANSPOSE_AVX2_INC_ +%define _TRANSPOSE_AVX2_INC_ -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" ; LOAD ALL 8 LANES FOR 8x8 32-BIT TRANSPOSE ; @@ -257,4 +257,4 @@ vshufps %%r3, %%r2, %%r3, 0xEE ; r3 = {d3 c3 b3 a3} %endmacro -%endif ;; _TRANSPOSE_AVX2_ASM_ +%endif ;; _TRANSPOSE_AVX2_INC_ diff --git a/lib/include/transpose_avx512.asm b/lib/include/transpose_avx512.inc similarity index 99% rename from lib/include/transpose_avx512.asm rename to lib/include/transpose_avx512.inc index 56c08d67542e28bbd891797827a2b120cb634f41..f0bac7d930a25295646f6b87a4d306edad80e1cb 100644 --- a/lib/include/transpose_avx512.asm +++ b/lib/include/transpose_avx512.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef _TRANSPOSE_AVX512_ASM_ -%define _TRANSPOSE_AVX512_ASM_ +%ifndef _TRANSPOSE_AVX512_INC_ +%define _TRANSPOSE_AVX512_INC_ -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" mksection .rodata default rel @@ -732,4 +732,4 @@ PSHUFFLE_TRANSPOSE_MASK2: dq 0x0000000000000002 vshufi64x2 %%r1, %%t0, %%t1, 0x44 ; 1 5 9 13 vshufi64x2 %%r3, %%t0, %%t1, 0xee ; 3 7 11 15 %endmacro -%endif ;; _TRANSPOSE_AVX512_ASM_ +%endif ;; _TRANSPOSE_AVX512_INC_ diff --git a/lib/include/transpose_sse.asm b/lib/include/transpose_sse.inc similarity index 95% rename from lib/include/transpose_sse.asm rename to lib/include/transpose_sse.inc index 74797788aa359f9f8c9caa69650afc09c233db30..5c4b26ea399867b4566be054a6a34449df4c3ef7 100644 --- a/lib/include/transpose_sse.asm +++ b/lib/include/transpose_sse.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,8 +25,8 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%ifndef _TRANSPOSE_SSE_ASM_ -%define _TRANSPOSE_SSE_ASM_ +%ifndef _TRANSPOSE_SSE_INC_ +%define _TRANSPOSE_SSE_INC_ ;; transpose r0, r1, r2, r3, t0, t1 ;; "transpose" data in {r0..r3} using temps {t0..t3} @@ -67,4 +67,4 @@ shufps %%t0, %%t1, 0x88 ; t0 = {d0 c0 b0 a0} %endmacro -%endif ;; _TRANSPOSE_SSE_ASM_ \ No newline at end of file +%endif ;; _TRANSPOSE_SSE_INC_ diff --git a/lib/include/wireless_common.h b/lib/include/wireless_common.h index 6851f9aeefa72dabf42dcec053dc973e92f3b023..2fb27b23b76dddcc75e1ba3e16a63a587eb0449c 100644 --- a/lib/include/wireless_common.h +++ b/lib/include/wireless_common.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -30,13 +30,13 @@ #include -#ifdef __x86_64__ +#ifndef __aarch64__ #ifdef LINUX #include #else #include #endif -#endif /* __x86_64__ */ +#endif /* __aarch64__ */ #define NUM_PACKETS_1 1 #define NUM_PACKETS_2 2 diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index a8624ee1db87a6093d40a82d864cb504841bcd93..43fff3b6fc85a8e784ac7fcda5485d8c29805b3c 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -43,9 +43,10 @@ #include "include/ipsec_ooo_mgr.h" #include "ipsec-mb.h" -#ifdef __x86_64__ +#ifndef __aarch64__ #include "immintrin.h" #endif + #include "include/wireless_common.h" /* 64 bytes of Keystream will be generated */ @@ -195,6 +196,7 @@ typedef struct zuc_key_16_s { /**< Array of pointers to 128-bit keys for the 16 packets */ } ZucKey16_t; +#ifndef __aarch64__ /** ****************************************************************************** * @@ -344,6 +346,9 @@ IMB_DLL_LOCAL void asm_Zuc256Initialization_4_avx(ZucKey4_t *pKeys, IMB_DLL_LOCAL void asm_ZucInitialization_8_avx2(ZucKey8_t *pKeys, const uint8_t *ivs, ZucState8_t *pState); +IMB_DLL_LOCAL void asm_ZucInitialization_8_gfni_avx2(ZucKey8_t *pKeys, + const uint8_t *ivs, + ZucState8_t *pState); /** ****************************************************************************** @@ -379,6 +384,11 @@ IMB_DLL_LOCAL void asm_Zuc256Initialization_8_avx2(ZucKey8_t *pKeys, ZucState8_t *pState, void *tags, const uint64_t tag_sz); +IMB_DLL_LOCAL void asm_Zuc256Initialization_8_gfni_avx2(ZucKey8_t *pKeys, + const uint8_t *ivs, + ZucState8_t *pState, + void *tags, + const uint64_t tag_sz); /** ****************************************************************************** @@ -635,6 +645,8 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream16B_4_avx(ZucState4_t *pState, *****************************************************************************/ IMB_DLL_LOCAL void asm_ZucGenKeystream32B_8_avx2(ZucState8_t *pState, uint32_t *pKeyStr[8]); +IMB_DLL_LOCAL void asm_ZucGenKeystream32B_8_gfni_avx2(ZucState8_t *pState, + uint32_t *pKeyStr[8]); /** ****************************************************************************** @@ -865,6 +877,8 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream4B_4_avx(ZucState4_t *pState, *****************************************************************************/ IMB_DLL_LOCAL void asm_ZucGenKeystream16B_8_avx2(ZucState8_t *pState, uint32_t *pKeyStr[8]); +IMB_DLL_LOCAL void asm_ZucGenKeystream16B_8_gfni_avx2(ZucState8_t *pState, + uint32_t *pKeyStr[8]); /** ****************************************************************************** @@ -888,6 +902,8 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream16B_8_avx2(ZucState8_t *pState, *****************************************************************************/ IMB_DLL_LOCAL void asm_ZucGenKeystream8B_8_avx2(ZucState8_t *pState, uint32_t *pKeyStr[8]); +IMB_DLL_LOCAL void asm_ZucGenKeystream8B_8_gfni_avx2(ZucState8_t *pState, + uint32_t *pKeyStr[8]); /** ****************************************************************************** @@ -911,6 +927,8 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream8B_8_avx2(ZucState8_t *pState, *****************************************************************************/ IMB_DLL_LOCAL void asm_ZucGenKeystream4B_8_avx2(ZucState8_t *pState, uint32_t *pKeyStr[8]); +IMB_DLL_LOCAL void asm_ZucGenKeystream4B_8_gfni_avx2(ZucState8_t *pState, + uint32_t *pKeyStr[8]); /** ****************************************************************************** @@ -1181,6 +1199,11 @@ IMB_DLL_LOCAL void asm_ZucCipher_8_avx2(ZucState8_t *pState, uint64_t *pOut[8], const uint16_t lengths[8], const uint64_t minLength); +IMB_DLL_LOCAL void asm_ZucCipher_8_gfni_avx2(ZucState8_t *pState, + const uint64_t *pIn[8], + uint64_t *pOut[8], + const uint16_t lengths[8], + const uint64_t minLength); /** ****************************************************************************** @@ -1486,6 +1509,14 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[8], const uint16_t lengthInBits[8], const void * const job_in_lane[8]); +IMB_DLL_LOCAL +void zuc_eia3_8_buffer_job_gfni_avx2(const void * const pKey[8], + const uint8_t *ivs, + const void * const pBufferIn[8], + uint32_t *pMacI[8], + const uint16_t lengthInBits[8], + const void * const job_in_lane[8]); + IMB_DLL_LOCAL void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[8], const uint8_t *ivs, @@ -1495,6 +1526,14 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[8], const void * const job_in_lane[8], const uint64_t tag_size); +IMB_DLL_LOCAL +void zuc256_eia3_8_buffer_job_gfni_avx2(const void * const pKey[8], + const uint8_t *ivs, + const void * const pBufferIn[8], + void *pMacI[8], + const uint16_t lengthInBits[8], + const void * const job_in_lane[8], + const uint64_t tag_size); /* the s-boxes */ extern const uint8_t S0[256]; extern const uint8_t S1[256]; @@ -1687,6 +1726,7 @@ void _zuc_eia3_8_buffer_avx2(const void * const pKey[8], const uint32_t lengthInBits[8], uint32_t *pMacI[8]); +#else /* __aarch64__ */ /* AARCH64 */ IMB_DLL_LOCAL void asm_ZucInitialization_aarch64(const void *pKey, @@ -2029,4 +2069,7 @@ void zuc256_eia3_4_buffer_job_aarch64_no_aesni(const void * const pKey[4], const uint32_t lengthInBits[4], const void * const job_in_lane[4], const uint64_t tag_size); +#endif /* __aarch64__ */ + #endif /* ZUC_INTERNAL_H_ */ + diff --git a/lib/include/zuc_sbox.inc b/lib/include/zuc_sbox.inc index 036b7b5da685fcb0766250e3d5888b7e479a0c9d..177b3383e1d2df39793fc50afec198831d4e65b6 100644 --- a/lib/include/zuc_sbox.inc +++ b/lib/include/zuc_sbox.inc @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -30,7 +30,7 @@ %define NO_AESNI_RENAME %include "include/aesni_emu.inc" -%include "include/os.asm" +%include "include/os.inc" mksection .rodata @@ -273,10 +273,11 @@ Rotl5_matrix: ; ; Compute 32 S0 box values from 32 bytes, stored in YMM register ; -%macro S0_comput_AVX2 3 +%macro S0_comput_AVX2 4 %define %%IN_OUT %1 ; [in/out] YMM reg with input values which will contain the output values %define %%YTMP1 %2 ; [clobbered] Temporary YMM register %define %%YTMP2 %3 ; [clobbered] Temporary YMM register +%define %%USE_GFNI %4 ; [in] If 1, GFNI can be used vpand %%YTMP1, %%IN_OUT, [rel High_nibble_mask] vpsrlq %%YTMP1, 4 ; x1 @@ -299,7 +300,11 @@ Rotl5_matrix: vpsllq %%IN_OUT, 4 vpor %%IN_OUT, %%YTMP1 ; t = (s << 4) | r +%if (%%USE_GFNI == 1) + vgf2p8affineqb %%IN_OUT, %%IN_OUT, [rel Rotl5_matrix], 0x00 +%else Rotl_5_AVX2 %%IN_OUT, %%YTMP1 +%endif %endmacro ; @@ -520,12 +525,19 @@ Rotl5_matrix: ; ; Compute 32 S1 box values from 32 bytes, stored in YMM register ; -%macro S1_comput_AVX2 4 +%macro S1_comput_AVX2 5 %define %%YIN_OUT %1 ; [in/out] YMM reg with input values which will contain the output values %define %%YTMP1 %2 ; [clobbered] Temporary YMM register %define %%YTMP2 %3 ; [clobbered] Temporary YMM register %define %%YTMP3 %4 ; [clobbered] Temporary YMM register +%define %%USE_GFNI %5 ; [in] If 1, GFNI can be used +%if (%%USE_GFNI == 1) + vgf2p8affineqb %%YIN_OUT, %%YIN_OUT, [rel Aes_to_Zuc], 0x00 + vpshufb %%YIN_OUT, [rel Shuf_mask] + vaesenclast %%YIN_OUT, %%YIN_OUT, [rel Cancel_aes] + vgf2p8affineqb %%YIN_OUT, %%YIN_OUT, [rel CombMatrix], 0x55 +%else ; USE_GFNI == 0 vmovdqa %%YTMP1, [rel Aes_to_Zuc_mul_low_nibble] vmovdqa %%YTMP2, [rel Aes_to_Zuc_mul_high_nibble] MUL_PSHUFB_AVX2 %%YIN_OUT, %%YTMP1, %%YTMP2, %%YTMP3 @@ -540,7 +552,7 @@ Rotl5_matrix: vmovdqa %%YIN_OUT, [rel Comb_matrix_mul_high_nibble] MUL_PSHUFB_AVX2 %%YTMP2, %%YTMP1, %%YIN_OUT, %%YTMP3 vpxor %%YIN_OUT, [rel Const_comb_matrix] - +%endif %endmacro ; diff --git a/lib/ipsec-mb.h b/lib/ipsec-mb.h index 75bd41f59e691fea3cd051eae168f95777b43f68..6c22e36666ee9426e67e14d2348ab921764cb830 100644 --- a/lib/ipsec-mb.h +++ b/lib/ipsec-mb.h @@ -91,8 +91,8 @@ typedef struct { /** * Library version */ -#define IMB_VERSION_STR "1.3.0" -#define IMB_VERSION_NUM 0x10300 +#define IMB_VERSION_STR "1.4.0" +#define IMB_VERSION_NUM 0x10400 /** * Macro to translate version number @@ -119,12 +119,15 @@ typedef struct { typedef enum { IMB_ARCH_NONE = 0, IMB_ARCH_NOAESNI, +#ifndef __aarch64__ IMB_ARCH_SSE, IMB_ARCH_AVX, IMB_ARCH_AVX2, IMB_ARCH_AVX512, +#else IMB_ARCH_AARCH64, IMB_ARCH_SVE256, +#endif IMB_ARCH_NUM, } IMB_ARCH; @@ -142,16 +145,30 @@ typedef enum { #define IMB_SHA384_DIGEST_SIZE_IN_BYTES 48 #define IMB_SHA512_DIGEST_SIZE_IN_BYTES 64 +#define IMB_MD5_DIGEST_SIZE_IN_BYTES 16 + #define IMB_SHA1_BLOCK_SIZE 64 /**< 512 bits is 64 byte blocks */ +#define IMB_SHA_224_BLOCK_SIZE 64 /**< 512 bits is 64 byte blocks */ #define IMB_SHA_256_BLOCK_SIZE 64 /**< 512 bits is 64 byte blocks */ #define IMB_SHA_384_BLOCK_SIZE 128 #define IMB_SHA_512_BLOCK_SIZE 128 +#define IMB_MD5_BLOCK_SIZE 64 + #define IMB_KASUMI_KEY_SIZE 16 #define IMB_KASUMI_IV_SIZE 8 #define IMB_KASUMI_BLOCK_SIZE 8 #define IMB_KASUMI_DIGEST_SIZE 4 +#define IMB_ZUC_KEY_LEN_IN_BYTES 16 +#define IMB_ZUC_IV_LEN_IN_BYTES 16 +#define IMB_ZUC256_KEY_LEN_IN_BYTES 32 +#define IMB_ZUC256_IV_LEN_IN_BYTES_MIN 23 +#define IMB_ZUC256_IV_LEN_IN_BYTES_MAX 25 +#define IMB_ZUC_DIGEST_LEN_IN_BYTES 4 +#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MIN 4 +#define IMB_ZUC256_DIGEST_LEN_IN_BYTES_MAX 16 + /** * Minimum Ethernet frame size to calculate CRC32 * Source Address (6 bytes) + Destination Address (6 bytes) + Type/Len (2 bytes) @@ -231,6 +248,7 @@ typedef enum { IMB_ERR_BURST_SIZE, IMB_ERR_BURST_OOO, IMB_ERR_SELFTEST, + IMB_ERR_BURST_SUITE_ID, /* add new error types above this comment */ IMB_ERR_MAX /* don't move this one */ } IMB_ERR; @@ -245,120 +263,6 @@ typedef enum { #endif #endif -/** - * Define enums from API v0.53, so applications that were using this version - * will still be compiled successfully. - * Note: this list has been extended with new names after version 0.55. - * This list does not need to be extended for new enums. - */ -#ifndef NO_COMPAT_IMB_API_053 -/* Previous cipher mode enums */ -#define CBC IMB_CIPHER_CBC -#define CNTR IMB_CIPHER_CNTR -#define NULL_CIPHER IMB_CIPHER_NULL -#define DOCSIS_SEC_BPI IMB_CIPHER_DOCSIS_SEC_BPI -#define GCM IMB_CIPHER_GCM -#define CUSTOM_CIPHER IMB_CIPHER_CUSTOM -#define DES IMB_CIPHER_DES -#define DOCSIS_DES IMB_CIPHER_DOCSIS_DES -#define CCM IMB_CIPHER_CCM -#define DES3 IMB_CIPHER_DES3 -#define PON_AES_CNTR IMB_CIPHER_PON_AES_CNTR -#define ECB IMB_CIPHER_ECB -#define CNTR_BITLEN IMB_CIPHER_CNTR_BITLEN - -/* Previous hash algo enums */ -#define SHA1 IMB_AUTH_HMAC_SHA_1 -#define SHA_224 IMB_AUTH_HMAC_SHA_224 -#define SHA_256 IMB_AUTH_HMAC_SHA_256 -#define SHA_384 IMB_AUTH_HMAC_SHA_384 -#define SHA_512 IMB_AUTH_HMAC_SHA_512 -#define AES_XCBC IMB_AUTH_AES_XCBC -#define MD5 IMB_AUTH_MD5 -#define NULL_HASH IMB_AUTH_NULL -#define AES_GMAC IMB_AUTH_AES_GMAC -#define CUSTOM_HASH IMB_AUTH_CUSTOM -#define AES_CCM IMB_AUTH_AES_CCM -#define AES_CMAC IMB_AUTH_AES_CMAC -#define PLAIN_SHA1 IMB_AUTH_SHA_1 -#define PLAIN_SHA_224 IMB_AUTH_SHA_224 -#define PLAIN_SHA_256 IMB_AUTH_SHA_256 -#define PLAIN_SHA_384 IMB_AUTH_SHA_384 -#define PLAIN_SHA_512 IMB_AUTH_SHA_512 -#define AES_CMAC_BITLEN IMB_AUTH_AES_CMAC_BITLEN -#define PON_CRC_BIP IMB_AUTH_PON_CRC_BIP - -/* Previous cipher direction enums */ -#define ENCRYPT IMB_DIR_ENCRYPT -#define DECRYPT IMB_DIR_DECRYPT - -/* Previous chain order enums */ -#define HASH_CIPHER IMB_ORDER_HASH_CIPHER -#define CIPHER_HASH IMB_ORDER_CIPHER_HASH - -/* Previous key size enums */ -#define AES_128_BYTES IMB_KEY_128_BYTES -#define AES_192_BYTES IMB_KEY_192_BYTES -#define AES_256_BYTES IMB_KEY_256_BYTES -#define IMB_KEY_AES_128_BYTES IMB_KEY_128_BYTES -#define IMB_KEY_AES_192_BYTES IMB_KEY_192_BYTES -#define IMB_KEY_AES_256_BYTES IMB_KEY_256_BYTES -#define AES_KEY_SIZE_BYTES IMB_KEY_SIZE_BYTES - -#define MB_MGR IMB_MGR -#define JOB_AES_HMAC IMB_JOB -#define JOB_STS IMB_STATUS -#define IMB_JOB_STS IMB_STATUS -#define JOB_CIPHER_MODE IMB_CIPHER_MODE -#define JOB_CIPHER_DIRECTION IMB_CIPHER_DIRECTION -#define JOB_HASH_ALG IMB_HASH_ALG -#define JOB_CHAIN_ORDER IMB_CHAIN_ORDER -#define MAX_JOBS IMB_MAX_JOBS - -#define STS_BEING_PROCESSED IMB_STATUS_BEING_PROCESSED -#define STS_COMPLETED_AES IMB_STATUS_COMPLETED_CIPHER -#define STS_COMPLETED_HMAC IMB_STATUS_COMPLETED_AUTH -#define STS_COMPLETED IMB_STATUS_COMPLETED -#define STS_INVALID_ARGS IMB_STATUS_INVALID_ARGS -#define STS_INTERNAL_ERROR IMB_STATUS_INTERNAL_ERROR -#define STS_ERROR IMB_STATUS_ERROR - -#define MAX_TAG_LEN IMB_MAX_TAG_LEN -#define GCM_IV_DATA_LEN IMB_GCM_IV_DATA_LEN -#define GCM_128_KEY_LEN IMB_GCM_128_KEY_LEN -#define GCM_192_KEY_LEN IMB_GCM_192_KEY_LEN -#define GCM_256_KEY_LEN IMB_GCM_256_KEY_LEN - -#define DES_KEY_SCHED_SIZE IMB_DES_KEY_SCHED_SIZE -#define DES_BLOCK_SIZE IMB_DES_BLOCK_SIZE - -#define AES_BLOCK_SIZE IMB_AES_BLOCK_SIZE - -#define SHA1_DIGEST_SIZE_IN_BYTES IMB_SHA1_DIGEST_SIZE_IN_BYTES -#define SHA224_DIGEST_SIZE_IN_BYTES IMB_SHA224_DIGEST_SIZE_IN_BYTES -#define SHA256_DIGEST_SIZE_IN_BYTES IMB_SHA256_DIGEST_SIZE_IN_BYTES -#define SHA384_DIGEST_SIZE_IN_BYTES IMB_SHA384_DIGEST_SIZE_IN_BYTES -#define SHA512_DIGEST_SIZE_IN_BYTES IMB_SHA512_DIGEST_SIZE_IN_BYTES - -#define SHA1_BLOCK_SIZE IMB_SHA1_BLOCK_SIZE -#define SHA_256_BLOCK_SIZE IMB_SHA_256_BLOCK_SIZE -#define SHA_384_BLOCK_SIZE IMB_SHA_384_BLOCK_SIZE -#define SHA_512_BLOCK_SIZE IMB_SHA_512_BLOCK_SIZE - -#define KASUMI_KEY_SIZE IMB_KASUMI_KEY_SIZE -#define KASUMI_IV_SIZE IMB_KASUMI_IV_SIZE -#define KASUMI_BLOCK_SIZE IMB_KASUMI_BLOCK_SIZE -#define KASUMI_DIGEST_SIZE IMB_KASUMI_DIGEST_SIZE - -#define DOCSIS_CRC32_MIN_ETH_PDU_SIZE IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE -#define DOCSIS_CRC32_TAG_SIZE IMB_DOCSIS_CRC32_TAG_SIZE - -/* Previous fields in IMB_JOB/JOB_AES_HMAC */ -#define aes_enc_key_expanded enc_keys -#define aes_dec_key_expanded dec_keys -#define aes_key_len_in_bytes key_len_in_bytes -#endif /* !NO_COMPAT_IMB_API_053 */ - typedef enum { IMB_CIPHER_CBC = 1, IMB_CIPHER_CNTR, @@ -447,6 +351,7 @@ typedef enum { } IMB_CHAIN_ORDER; typedef enum { + IMB_KEY_64_BYTES = 8, IMB_KEY_128_BYTES = 16, IMB_KEY_192_BYTES = 24, IMB_KEY_256_BYTES = 32 @@ -646,6 +551,9 @@ typedef struct IMB_JOB { /**< Pointer to next IV (last ciphertext block) */ } CBCS; /**< CBCS specific fields */ } cipher_fields; /**< Cipher algorithm-specific fields */ + + void *suite_id[4]; /**< see imb_set_session() */ + uint32_t session_id; /**< see imb_set_session() */ } IMB_JOB; @@ -764,10 +672,16 @@ struct gcm_key_data { } avx2_avx512; struct { /** + * Start with: * (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., - * (Hashkey^48<<1 mod poly) + * (Hashkey^32<<1 mod poly) + * + * Followed by: + * (HashKey<<1 mod poly) x POLY, + * (HashKey^2<<1 mod poly) x POLY, ..., + * (Hashkey^32<<1 mod poly) x POLY */ - uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 48]; + uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 32 * 2]; } vaes_avx512; } ghash_keys; } @@ -981,7 +895,7 @@ typedef void (*snow3g_f8_4_buffer_multikey_t)(const snow3g_key_schedule_t *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t); -#endif //__aarch64__ +#endif typedef void (*snow3g_f8_8_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const void *, @@ -1027,6 +941,10 @@ typedef uint32_t (*hec_32_t)(const uint8_t *); typedef uint64_t (*hec_64_t)(const uint8_t *); typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); + +typedef void (*aes_ecb_quic_t)(const void *, const void *, + void *out, uint64_t); + /* Multi-buffer manager flags passed to alloc_mb_mgr() */ #define IMB_FLAG_SHANI_OFF (1ULL << 0) /**< disable use of SHANI extension */ @@ -1065,6 +983,7 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_FEATURE_AESNI_EMU (1ULL << 19) #define IMB_FEATURE_SELF_TEST (1ULL << 20) /* self-test feature present */ #define IMB_FEATURE_SELF_TEST_PASS (1ULL << 21) /* self-test passed */ +#define IMB_FEATURE_AVX_IFMA (1ULL << 22) /** * CPU flags needed for each implementation @@ -1086,6 +1005,7 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); IMB_FEATURE_GFNI) #define IMB_CPUFLAGS_AVX_T2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_SHANI | \ IMB_FEATURE_GFNI) +#define IMB_CPUFLAGS_AVX2_T3 (IMB_CPUFLAGS_AVX2_T2 | IMB_FEATURE_AVX_IFMA) #define IMB_FEATURE_AARCH64 (1ULL << 32) #define IMB_FEATURE_ASIMD (1ULL << 33) @@ -1170,6 +1090,7 @@ typedef struct IMB_MGR { zuc_eea3_4_buffer_t eea3_4_buffer; zuc_eea3_n_buffer_t eea3_n_buffer; zuc_eia3_1_buffer_t eia3_1_buffer; + #ifdef __aarch64__ zuc256_eea3_1_buffer_t zuc256_eea3_1_buffer; zuc256_eea3_n_buffer_t zuc256_eea3_n_buffer; @@ -1196,7 +1117,7 @@ typedef struct IMB_MGR { snow3g_f8_n_buffer_t snow3g_f8_n_buffer; #ifdef __aarch64__ snow3g_f8_4_buffer_multikey_t snow3g_f8_4_buffer_multikey; -#endif //__aarch64__ +#endif snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey; snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey; snow3g_f9_1_buffer_t snow3g_f9_1_buffer; @@ -1248,6 +1169,12 @@ typedef struct IMB_MGR { submit_cipher_burst_t submit_cipher_burst_nocheck; submit_hash_burst_t submit_hash_burst; submit_hash_burst_t submit_hash_burst_nocheck; + aes_cfb_t aes256_cfb_one; + + aes_ecb_quic_t aes_ecb_128_quic; + aes_ecb_quic_t aes_ecb_256_quic; + + void (*set_suite_id)(struct IMB_MGR *, IMB_JOB *); /* in-order scheduler fields */ int earliest_job; /**< byte offset, -1 if none */ @@ -1688,12 +1615,16 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); ((_mgr)->get_next_burst((_mgr), (_n_jobs), (_jobs))) /** - * Submit multiple jobs to be processed after validating. + * @brief Submit multiple jobs to be processed after validating. + * + * Prior to submission, \a _jobs need to be initialized with correct + * crypto job parameters and followed with a call to imb_set_session(). * * @param [in,out] _mgr Pointer to initialized IMB_MGR structure * @param [in] _n_jobs Number of jobs to submit for processing * @param [in,out] _jobs In: List of pointers to jobs for submission * Out: List of pointers to completed jobs + * @see imb_set_session() * * @return Number of completed jobs or zero on error. * If zero, imb_get_errno() can be used to check for potential @@ -1703,12 +1634,16 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); ((_mgr)->submit_burst((_mgr), (_n_jobs), (_jobs))) /** - * Submit multiple jobs to be processed without validating. + * @brief Submit multiple jobs to be processed without validating. + * + * Prior to submission \a _jobs need to be initialized with correct + * crypto job parameters and followed with call to imb_set_session(). * * @param [in,out] _mgr Pointer to initialized IMB_MGR structure * @param [in] _n_jobs Number of jobs to submit for processing * @param [in,out] _jobs In: List of pointers to jobs for submission * Out: List of pointers to completed jobs + * @see imb_set_session() * * @return Number of completed jobs or zero on error */ @@ -1984,6 +1919,22 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); #define IMB_AES128_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len) \ ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len))) +/** + * @brief AES-CFB-256 Encrypt/Decrypt up to one block. + * + * Processes only one buffer at a time. + * Designed to manage partial blocks of DOCSIS 3.1 SEC BPI. + * + * @param [in] _mgr Pointer to multi-buffer structure + * @param [out] _dst Plaintext/Ciphertext output + * @param [in] _src Plaintext/Ciphertext input + * @param [in] _iv Pointer to 16 byte IV + * @param [in] _exp_key Pointer to expanded AES keys + * @param [in] _len Length of data in bytes + */ +#define IMB_AES256_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len) \ + ((_mgr)->aes256_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len))) + /* AES-GCM API's */ #define IMB_AES128_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \ _aadl, _tag, _tagl) \ @@ -2517,7 +2468,7 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); (_src2), (_dst2), (_len2), \ (_src3), (_dst3), (_len3), \ (_src4), (_dst4), (_len4))) -#endif //__aarch64__ +#endif /** ******************************************************************************* @@ -2772,230 +2723,35 @@ IMB_DLL_EXPORT int des_key_schedule(uint64_t *ks, const void *key); /** - * Authenticate variable sized data with SHA1. + * @brief Ipad Opad padding for HMAC * - * @param[in] data Data buffer - * @param[in] length Length of data in bytes for authentication. - * @param[out] digest Digest output (20 bytes) - */ -IMB_DLL_EXPORT void sha1_sse(const void *data, const uint64_t length, - void *digest); - -/** - * @copydoc sha1_sse - */ -IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha1_sse - */ -IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha1_sse - */ -IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length, - void *digest); - -/** - * Authenticate 64-byte data buffer with SHA1. - * - * @param[in] data 64-byte data buffer - * @param[out] digest Digest output (20 bytes) - */ -IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest); -/** - * @copydoc sha1_one_block_sse - */ -IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest); -/** - * @copydoc sha1_one_block_sse - */ -IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest); -/** - * @copydoc sha1_one_block_sse - */ -IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest); - -/** - * Authenticate variable sized data with SHA224. - * - * @param[in] data Data buffer - * @param[in] length Length of data in bytes for authentication. - * @param[out] digest Digest output (28 bytes) - */ -IMB_DLL_EXPORT void sha224_sse(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha224_sse - */ -IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha224_sse - */ -IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha224_sse - */ -IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length, - void *digest); - -/** - * Authenticate 64-byte data buffer with SHA224. - * - * @param[in] data 64-byte data buffer - * @param[out] digest Digest output (28 bytes) - */ -IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest); -/** - * @copydoc sha224_one_block_sse - */ -IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest); -/** - * @copydoc sha224_one_block_sse - */ -IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest); -/** - * @copydoc sha224_one_block_sse - */ -IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest); - -/** - * Authenticate variable sized data with SHA256. - * - * @param[in] data Data buffer - * @param[in] length Length of data in bytes for authentication. - * @param[out] digest Digest output (32 bytes) - */ -IMB_DLL_EXPORT void sha256_sse(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha256_sse - */ -IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha256_sse + * @param [in] mb_mgr Pointer to initialized IMB_MGR structure + * @param [in] sha_type Type of HMAC_SHA from IMB_HASH_ALG enum + * @param [in] pkey Pointer to a HMAC key + * @param [in] key_len Length of the HMAC key + * @param [out] ipad_hash Block-sized inner padding + * @param [out] opad_hash Block-sized outer padding */ -IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha256_sse - */ -IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length, - void *digest); - -/** - * Authenticate 64-byte data buffer with SHA256. - * - * @param[in] data 64-byte data buffer - * @param[out] digest Digest output (32 bytes) - */ -IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest); -/** - * @copydoc sha256_one_block_sse - */ -IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest); -/** - * @copydoc sha256_one_block_sse - */ -IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest); -/** - * @copydoc sha256_one_block_sse - */ -IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest); - -/** - * Authenticate variable sized data with SHA384. - * - * @param[in] data Data buffer - * @param[in] length Length of data in bytes for authentication. - * @param[out] digest Digest output (48 bytes) - */ -IMB_DLL_EXPORT void sha384_sse(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha384_sse - */ -IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha384_sse - */ -IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha384_sse - */ -IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length, - void *digest); - -/** - * Authenticate 128-byte data buffer with SHA384. - * - * @param[in] data 64-byte data buffer - * @param[out] digest Digest output (48 bytes) - */ -IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest); -/** - * @copydoc sha384_one_block_sse - */ -IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest); -/** - * @copydoc sha384_one_block_sse - */ -IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest); -/** - * @copydoc sha384_one_block_sse - */ -IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest); +IMB_DLL_EXPORT void +imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, + const void *pkey, const size_t key_len, + void *ipad_hash, void *opad_hash); /** - * Authenticate variable sized data with SHA512. + * @brief DES-CFB Encrypt/Decrypt up to one block. * - * @param[in] data Data buffer - * @param[in] length Length of data in bytes for authentication. - * @param[out] digest Digest output (64 bytes) - */ -IMB_DLL_EXPORT void sha512_sse(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha512_sse - */ -IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha512_sse - */ -IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length, - void *digest); -/** - * @copydoc sha512_sse - */ -IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length, - void *digest); - -/** - * Authenticate 64-byte data buffer with SHA512. + * Processes only one buffer at a time. + * Designed to manage partial blocks of DOCSIS 3.1 SEC BPI. * - * @param[in] data 128-byte data buffer - * @param[out] digest Digest output (64 bytes) - */ -IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest); -/** - * @copydoc sha512_one_block_sse - */ -IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest); -/** - * @copydoc sha512_one_block_sse - */ -IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest); -/** - * @copydoc sha512_one_block_sse + * @param [out] out Plaintext/Ciphertext output + * @param [in] in Plaintext/Ciphertext input + * @param [in] iv Pointer to 8 byte IV + * @param [in] ks Pointer to DES key schedule + * @param [in] len Length of data in bytes */ -IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest); +IMB_DLL_EXPORT void +des_cfb_one(void *out, const void *in, const uint64_t *iv, + const uint64_t *ks, const int len); /** * Authenticate 64-byte data buffer with MD5. @@ -3215,39 +2971,6 @@ IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1, */ IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1, void *key2); -/** - * @brief AES-CFB-128 Encrypt/Decrypt up to one block. - * - * Processes only one buffer at a time. - * Designed to manage partial blocks of DOCSIS 3.1 SEC BPI. - * - * @param [out] out Plaintext/Ciphertext output - * @param [in] in Plaintext/Ciphertext input - * @param [in] iv Pointer to 16 byte IV - * @param [in] keys Pointer to expanded AES keys - * @param [in] len Length of data in bytes - */ -IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in, - const void *iv, const void *keys, - uint64_t len); -/** - * @copydoc aes_cfb_128_one_sse - */ -IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in, - const void *iv, const void *keys, - uint64_t len); -/** - * @copydoc aes_cfb_128_one_sse - */ -IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in, - const void *iv, const void *keys, - uint64_t len); -/** - * @copydoc aes_cfb_128_one_sse - */ -IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in, - const void *iv, const void *keys, - uint64_t len); /* * Direct GCM API. @@ -4197,6 +3920,105 @@ IMB_DLL_EXPORT int snow3g_f9_iv_gen(const uint32_t count, */ IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size); +/** + * @brief Batch of GCM encrypt/decrypt operations with the same key + * + * @note IV length of 12 bytes is assumed. + * @note If used out of place then AAD needs to be copied by the caller. + * @note For more info on key_data refer to IMB_AES128/192/256_GCM_PRE() API’s + * + * @param [in] state pointer to IMB_MGR + * @param [in] key_data initialized key data (AES keys and hash keys) + * @param [in] key_size key size (in bytes, see IMB_KEY_128_BYTES etc.) + * @param [in] cipher_dir cipher direction (IMB_DIR_ENCRYPT / DECRYPT) + * @param [out] dst_ptr_array array with destination pointers + * @param [in] src_ptr_array array with source pointers + * @param [in] len_array array with message lengths in bytes + * @param [in] iv_ptr_array array with IV pointers + * @param [in] aad_ptr_array array with AAD pointers + * @param [in] aad_len AAD length in bytes + * @param [out] tag_ptr_array array with authentication TAG pointers + * @param [in] tag_len authentication TAG length in bytes + * @param [in] num_packets number of packets in this batch + */ +IMB_DLL_EXPORT void +imb_quic_aes_gcm(IMB_MGR *state, + const struct gcm_key_data *key_data, + const IMB_KEY_SIZE_BYTES key_size, + const IMB_CIPHER_DIRECTION cipher_dir, + void *dst_ptr_array[], + const void * const src_ptr_array[], + const uint64_t len_array[], + const void * const iv_ptr_array[], + const void * const aad_ptr_array[], + const uint64_t aad_len, + void *tag_ptr_array[], + const uint64_t tag_len, + const uint64_t num_packets); + +/** + * @brief Batch of AES-ECB encrypt/decrypt operations with the same key + * + * Sample size is fixed to 16 bytes (read from source pointers). + * Mask output size is fixed to 5 bytes (written to destination pointer). + * Cipher direction is fixed to ENCRYPT. + * + * @param [in] state pointer to IMB_MGR + * @param [in] exp_key_data expanded AES encrypt keys + * @param [out] dst_ptr_array array with destination pointers + * @param [in] src_ptr_array array with source sample pointers + * @param [in] num_packets number of packets in this batch + * @param [in] key_size key size (in bytes, see IMB_KEY_128_BYTES etc.) + */ +IMB_DLL_EXPORT void +imb_quic_hp_aes_ecb(IMB_MGR *state, + const void *exp_key_data, + void *dst_ptr_array[], + const void * const src_ptr_array[], + const uint64_t num_packets, + const IMB_KEY_SIZE_BYTES key_size); + +/** + * @brief Sets up suite_id and session_id fields for selected cipher suite in + * provided \a job structure + * + * This is mandatory operation for BURST API as suite_id is used to speed up + * job dispatch process. + * This operation is optional but helpful for JOB API use case. + * + * 'session_id' field is for application use to optimize job set up process. + * If JOB structure provided by library for a new operation has same session ID + * as required for the next operation then only message pointers and sizes + * need to be set up by the application. All other session fields are guaranteed + * to be unmodified by the library: + * - cipher mode + * - cipher direction + * - hash algorithm + * - key size + * - encrypt & decrypt key pointers + * - suite_id + * If allocated JOB structure contains different session ID then + * all required session and crypto operation fields need to be set up. + * + * In connection oriented applications, a template filled-in job structure + * can be cached within connection structure and reused in submit operations. + * + * For given set of parameters: cipher mode, cipher key size, + * cipher direction and authentication mode, suite_id field is the same. + * + * @see IMB_SUBMIT_BURST() + * @see IMB_SUBMIT_BURST_NOCHECK() + * @see IMB_SUBMIT_JOB() + * @see IMB_SUBMIT_JOB_NOCHECK() + * + * @param [in] state pointer to IMB_MGR + * @param [in,out] job pointer to prepared JOB structure + * + * @return Session ID value + * @retval 0 on error + */ +IMB_DLL_EXPORT uint32_t imb_set_session(IMB_MGR *state, IMB_JOB *job); + #ifdef __cplusplus } #endif diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index 903b3368826d3574ba38393a8a7a6c29627c6fe3..97a31cdb6481e54c99cde86c371ec1a53a79030f 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -1,4 +1,4 @@ -; Copyright (c) 2017-2022, Intel Corporation +; Copyright (c) 2017-2023, Intel Corporation ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -677,4 +677,36 @@ EXPORTS get_completed_job_avx2_t1 @651 get_completed_job_avx2_t2 @652 get_completed_job_avx512_t1 @653 - get_completed_job_avx512_t2 @654 \ No newline at end of file + get_completed_job_avx512_t2 @654 + get_next_burst_avx2_t3 @655 + submit_burst_avx2_t3 @656 + submit_burst_nocheck_avx2_t3 @657 + flush_burst_avx2_t3 @658 + submit_cipher_burst_avx2_t3 @659 + submit_cipher_burst_nocheck_avx2_t3 @660 + submit_hash_burst_avx2_t3 @661 + submit_hash_burst_nocheck_avx2_t3 @662 + flush_job_avx2_t3 @663 + queue_size_avx2_t3 @664 + submit_job_avx2_t3 @665 + submit_job_nocheck_avx2_t3 @666 + get_next_job_avx2_t3 @667 + get_completed_job_avx2_t3 @668 + des_cfb_one @669 + aes_cfb_256_one_sse @670 + aes_cfb_256_one_avx @671 + aes_cfb_256_one_avx2 @672 + aes_cfb_256_one_avx512 @673 + aes_cfb_256_one_sse_no_aesni @674 + imb_quic_aes_gcm @675 + imb_quic_hp_aes_ecb @676 + imb_hmac_ipad_opad @677 + imb_set_session @678 + ghash_pre_avx_gen4 @679 + ghash_pre_avx512 @680 + sha1_sse_shani @681 + sha1_one_block_sse_shani @682 + sha224_sse_shani @683 + sha224_one_block_sse_shani @684 + sha256_sse_shani @685 + sha256_one_block_sse_shani @686 diff --git a/lib/libipsec-mb.7 b/lib/libipsec-mb.7 index abe8d1674fc8c82c698c1d3bd31658338719fc88..432def7adb8e901828e7a07a280f9680ca822056 100644 --- a/lib/libipsec-mb.7 +++ b/lib/libipsec-mb.7 @@ -1,5 +1,5 @@ .\" Hey, EMACS: -*- nroff -*- -.\" Copyright (c) 2018-2022, Intel Corporation, written by Tomasz Kantecki +.\" Copyright (c) 2018-2023, Intel Corporation, written by Tomasz Kantecki .\" .\" .\" %%%LICENSE_START(VERBATIM) diff --git a/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm index 7fef27fe1373ff517de35d2e5f36ef55dbef0765..7b164d954e6c35c4bdb14c9ac8085cec34e5953c 100644 --- a/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm index c925419ac617026f57a2f28e8c2dfb4b89764477..d51f0e45a028e9eec73e54bf24c7f7f931206b07 100644 --- a/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm index 999176ea6252541ea3f27a15b468949f7c065b77..d4e523827d1e15d1b57ab353d2d826e3fb826ca9 100644 --- a/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm index c30441adda3ad7277811e730fe0d9a01abdc1e24..3bdb131e3d0a7ac6cc7a1a33c20e248165f1b807 100644 --- a/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm index ce0f0cb11d634d10c563228a40200b2e92be1d36..a2002a3dc1d05d05044e785c7572f998e9717968 100644 --- a/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm b/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm index 5e001722cfefbd38edcd6b617bae3a65baf6a255..12fa2f69808b4a2143ec5ad81f6b17fdfbfc7e74 100644 --- a/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm b/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm index e3e5d0f4989551c536c6c51ca3ba4da2b8c8d995..08c6d08bd59bdf5ea7b95605aa7471b5899015fb 100644 --- a/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm index 20c4ff6c04a671605f04cd70f7c2d923fc7fd741..eed20f350ea4af856104cbbb975a09ee0986dc31 100644 --- a/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm index b4d9c7aed3538e24ae5eea468b52d38094126366..c1d37a544111107fd5da5f98eb4d2ff1384d3c6e 100644 --- a/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm index 720ccb507515a457001b095ba16bd9c9a55da1d9..897ba0e5a90c37cd9bb0aca08f7c2c967d842586 100644 --- a/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm b/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm index 4f9cb9adb34df53ef3785dff8cbcdc97f8c7ad01..5d45b9d0d8874281fe40e6622a8518486050063f 100644 --- a/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm index a4f1d010295af8841e6ef49c1e5f0ae266f68492..7a91fa658bf36e7e8975a1e6cdfcab283c28ffeb 100644 --- a/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm index 4c6ae009cba2417d87e66f270dd792352c6994f6..ce8e21ed2af3ed026f1e07488771ce670ce1e01e 100644 --- a/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm b/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm index ee31c351202497111dbef0731542d2c5519995a6..0160a305a24dc1a6a99b169d71780c7ac7d5b048 100644 --- a/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm b/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm index 00811ff12f21244b698278bbc227cbf2116ee1bd..5529e8837be8b26c23d2f41456c6479c2b0825ef 100644 --- a/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm b/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm index 8057968fe7b7dd56bacce40ad7de77b8360ac75e..d0e12ed896fd5a4219abbd22f8d951734a694d3f 100644 --- a/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes_cfb_sse_no_aesni.asm b/lib/no-aesni/aes_cfb_sse_no_aesni.asm index 8c033ce44a96c63cfcdc010eb7ca819743374373..89108d6ce477ec1da536a2d490f7430745b1f4c1 100644 --- a/lib/no-aesni/aes_cfb_sse_no_aesni.asm +++ b/lib/no-aesni/aes_cfb_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm b/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm index f5072f67fc5eb6dcf74b20494b0defb1c053f95b..37928841a56e3d20edf5c9d1a91eb019d668e428 100644 --- a/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -32,4 +32,7 @@ %define AES_ECB_DEC_128 aes_ecb_dec_128_sse_no_aesni %define AES_ECB_DEC_192 aes_ecb_dec_192_sse_no_aesni %define AES_ECB_DEC_256 aes_ecb_dec_256_sse_no_aesni +%define AES_ECB_QUIC_ENC_128 aes_ecb_quic_enc_128_sse_no_aesni +%define AES_ECB_QUIC_ENC_256 aes_ecb_quic_enc_256_sse_no_aesni %include "sse_t1/aes128_ecb_by4_sse.asm" +%include "sse_t1/aes_ecb_quic_x8_sse.asm" diff --git a/lib/no-aesni/aesni_emu.c b/lib/no-aesni/aesni_emu.c index 29492234837ac4ed82ebf945886679f942e1e732..3b8aa1c01f3b4636c42cce58f73ff0f0a7807df6 100644 --- a/lib/no-aesni/aesni_emu.c +++ b/lib/no-aesni/aesni_emu.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -28,6 +28,7 @@ /* ========================================================================== */ /* AESNI emulation API and helper functions */ /* ========================================================================== */ + #define AESNI_EMU #include "ipsec-mb.h" @@ -36,16 +37,15 @@ #ifdef __aarch64__ #include "aarch64/constant_lookup_aarch64.h" #include -#endif /* __aarch64__ */ - -#ifdef __86_x64__ +#else /* __aarch64__ */ #include "include/constant_lookup.h" + #ifdef LINUX #include #else #include #endif -#endif /* __x86_64__ */ +#endif /* __aarch64__ */ static const DECLARE_ALIGNED(uint8_t aes_sbox[16][16], 16) = { { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, diff --git a/lib/no-aesni/crc16_x25_sse_no_aesni.asm b/lib/no-aesni/crc16_x25_sse_no_aesni.asm index 9c46b805608070de418e999a53469d59daa37cf8..bc3f1e9ae0a863291878f45a948dd98712bf5934 100644 --- a/lib/no-aesni/crc16_x25_sse_no_aesni.asm +++ b/lib/no-aesni/crc16_x25_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/crc32_by8_sse_no_aesni.asm b/lib/no-aesni/crc32_by8_sse_no_aesni.asm index 141c626bbefa177b2bf5e8b52563641585f01361..5f849479d698acda6e97a809103a6818626b21c3 100644 --- a/lib/no-aesni/crc32_by8_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_by8_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/crc32_fp_sse_no_aesni.asm b/lib/no-aesni/crc32_fp_sse_no_aesni.asm index 4dccb4e9c94d0ecdc20e54f2aa6ea4b0688c2c95..2b4405bb8769ef4bc636a4d1498574eab18c9130 100644 --- a/lib/no-aesni/crc32_fp_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_fp_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/crc32_iuup_sse_no_aesni.asm b/lib/no-aesni/crc32_iuup_sse_no_aesni.asm index 663acc2b7f0067ead491117c5156efb9d29e3092..3524d85f04985c3174001f51d5d9b19ffe95a1c2 100644 --- a/lib/no-aesni/crc32_iuup_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_iuup_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/crc32_lte_sse_no_aesni.asm b/lib/no-aesni/crc32_lte_sse_no_aesni.asm index 9b21b49e0901ebc3184c8b30928b2e8bf3cb3069..7c3af7a68b371406f94fd1d8aff4c085cafee649 100644 --- a/lib/no-aesni/crc32_lte_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_lte_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm b/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm index 53027208dfbfb42238a89cb2748ce0c5fe8b6b0e..fa76463bde23359fc0d4d00a433453e4cd64ef58 100644 --- a/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/crc32_sctp_sse_no_aesni.asm b/lib/no-aesni/crc32_sctp_sse_no_aesni.asm index 2f7d9b6b3f4573bc24926aa9811e43fc4e9e5593..3399cbcb042f815322e7993b732c6f0d9d6906fe 100644 --- a/lib/no-aesni/crc32_sctp_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_sctp_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/crc32_wimax_sse_no_aesni.asm b/lib/no-aesni/crc32_wimax_sse_no_aesni.asm index 0c6948296e881cee74666e5c947d292fa138d94c..ea5b9ff826bb85f48bf1bf4a58dae19c85f874ed 100644 --- a/lib/no-aesni/crc32_wimax_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_wimax_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm b/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm index c0f9f0fcbdea421ef0ec691807ad599f787dcab1..176eb4bfdd402509bdae99fe13a1694540028b65 100644 --- a/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm +++ b/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/gcm128_api_sse_no_aesni.asm b/lib/no-aesni/gcm128_api_sse_no_aesni.asm index e116bcf6c4671fc13acdb35f4c191b3f1eca4798..741e841e28d879b2d87bfb973870c65d299a542b 100644 --- a/lib/no-aesni/gcm128_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm128_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022 Intel Corporation All rights reserved. +; Copyright(c) 2018-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm b/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm index 8fa5c283519de20a6a1e783729c8cf68bd328a81..aa380e6416de9532aed526e33dcb85e12aba1363 100644 --- a/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022 Intel Corporation All rights reserved. +; Copyright(c) 2018-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm b/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm index a6750c2fb39d4539eb56cf511b864782ce7a8253..0d3ecb82b494a8571e4209f3249159fd0dfd4672 100644 --- a/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022 Intel Corporation All rights reserved. +; Copyright(c) 2018-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm192_api_sse_no_aesni.asm b/lib/no-aesni/gcm192_api_sse_no_aesni.asm index efe16366de1e49c461c6f10d450a48281426a27c..25d6d14a5f63b8bac3908c9cf1e4251fbd8c42dd 100644 --- a/lib/no-aesni/gcm192_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm192_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm b/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm index 8c6345beeed76c4283a73113a4e9cd875ebc5112..331bbf4cadeb5785b8c9ca940e40a58ad339859b 100644 --- a/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm b/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm index 7a0a962ecc4491a6e9f5fbf626160c1378d3d1e0..c1afbbea30d129c7317abb2394b8573d5114d6af 100644 --- a/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022, Intel Corporation All rights reserved. +; Copyright(c) 2018-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm256_api_sse_no_aesni.asm b/lib/no-aesni/gcm256_api_sse_no_aesni.asm index 320ec91e99e13406ee75232963d4e7b5e4e9f666..cf050440c98321d8b83d80e006abcaf90ae6b7fe 100644 --- a/lib/no-aesni/gcm256_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm256_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022 Intel Corporation All rights reserved. +; Copyright(c) 2018-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm b/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm index 2507431982774c2903ace466720d988c6a447baa..f6dfbe39afe9be9563b87d6d28ddff9c20b5b34f 100644 --- a/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022 Intel Corporation All rights reserved. +; Copyright(c) 2018-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm b/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm index 713924e701782435c13c5109c893bdc09b942550..c0a4535c7f03906c65e5ce87f5084c87453f0cfe 100644 --- a/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2018-2022 Intel Corporation All rights reserved. +; Copyright(c) 2018-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm index f0cc5e804a91ecf807be299d9f342b28982c3616..d955d1c3f390eabd2850553272d86756a1258d37 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm index 76332f3788d7f904d81bb09bf46226eeea3086c7..efe896e0e0e4e2cb62c4d97632bac24d28b02f38 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm index a6bca7a0385cb8f583dcec8b2ab2ec50bcf583dd..4d55948ff3a439cfd92cc6276b8f7ee5f266404e 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm index 1793d305e0546f19b383631a68ed1ca0fd5573f7..83da5c117a09792ceacaccd668c4e0b19af59300 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm index e88628397f31d2eb1d293b8670b0e93eeb911c9d..6a28f6c78683467f9ee331d60a90129bd20ac95c 100644 --- a/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm index dfd49155bfd851d5b528d8c901ef8022b99aac7f..13f1a52e515f24dbca2eedf6630ac2a7447e8013 100644 --- a/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm index 116f0ab154ab49ed8e22df06095a7374d9407a20..9034f35a7d05656b2f2fd61050d6076a839467d6 100644 --- a/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm index 20b036ea2dcab9965e13dac38d6d3033df7a184d..3b7b0e21711b307d8bae9c4f7490ddcd67ff77da 100644 --- a/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm index e700ca6225a6acd412985735e9206d0179e20f7b..b78423adb33690c7178004e3a68b5dae797116d1 100644 --- a/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm index 50c86fe776fe4fea31edb1ed49e4fccedaed9017..99c908b40363692b769de75a43a3c3d55232e53b 100644 --- a/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm index 4256da5f7b5232927b300933b3af9cebf7040046..66a165970d84cee515e4290e250f5e76d4341d18 100644 --- a/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm index 46de48dad4a2260eaf854613c9ef82832e1c9ec2..ef98ea86977195c858a59add3ed892c3f3330c94 100644 --- a/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm index 02af37da7e59956f9526fd0475d185a744636f3c..06147ebdb7865a6f785ef719266492996905afc9 100644 --- a/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm index 7d543d0ea3b216b1c5b86844e458caacaaa5f923..f7bcc99ec58bf5ebcc901b90bfaa9e050daf8333 100644 --- a/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 0b3a15865d1c90482c05c0951260dae9c1a0a877..48ff91114afd3b7fdc985f98c0024a2f13aadb53 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -166,6 +166,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_no_aesni #define SUBMIT_HASH_BURST submit_hash_burst_sse_no_aesni #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_no_aesni +#define SET_SUITE_ID_FN set_suite_id_sse_no_aesni #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni @@ -370,6 +371,7 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) state->sha512 = sha512_sse; state->md5_one_block = md5_one_block_sse; state->aes128_cfb_one = aes_cfb_128_one_sse_no_aesni; + state->aes256_cfb_one = aes_cfb_256_one_sse_no_aesni; state->eea3_1_buffer = zuc_eea3_1_buffer_sse_no_aesni; state->eea3_4_buffer = zuc_eea3_4_buffer_sse_no_aesni; @@ -465,6 +467,11 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_sse; state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_sse; state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_sse; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_sse_no_aesni; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_sse_no_aesni; + + state->set_suite_id = SET_SUITE_ID_FN; } void diff --git a/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm index 2b3009c232262615bc368471afaef64c1457003b..5a15500e583336b02ed256652ef263b5a5072cfe 100644 --- a/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/pon_sse_no_aesni.asm b/lib/no-aesni/pon_sse_no_aesni.asm index 65170f1d7e05419129cea7e03cf1d95331bbdf9a..a526bf8478f179d8581e1832be138f3ed769de17 100644 --- a/lib/no-aesni/pon_sse_no_aesni.asm +++ b/lib/no-aesni/pon_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/snow3g_sse_no_aesni.c b/lib/no-aesni/snow3g_sse_no_aesni.c index 930e5cacb38ea768a0c6e8e81c224761301fea26..7112221a85dab96bf348e6d701958a3fba3feb85 100644 --- a/lib/no-aesni/snow3g_sse_no_aesni.c +++ b/lib/no-aesni/snow3g_sse_no_aesni.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm b/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm index 676912cfbade1b28c70fca95fcce66b79cd12e47..0ef6feddf8f661ac9300c4cd526850cab77c937b 100644 --- a/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm +++ b/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/snow_v_sse_no_aesni.asm b/lib/no-aesni/snow_v_sse_no_aesni.asm index 65559065f64acd73ced56ebb0bcdc849c60a6360..3f9158e66d115339d41d57e4c0d9ee530e5378aa 100644 --- a/lib/no-aesni/snow_v_sse_no_aesni.asm +++ b/lib/no-aesni/snow_v_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/zuc_sse_no_aesni.asm b/lib/no-aesni/zuc_sse_no_aesni.asm index d43e99d7e1b74e72f9572f1add35e141ad4d2a9d..87ccf5b87080c1d610a21a83840ca0dee5748946 100644 --- a/lib/no-aesni/zuc_sse_no_aesni.asm +++ b/lib/no-aesni/zuc_sse_no_aesni.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/no-aesni/zuc_top_sse_no_aesni.c b/lib/no-aesni/zuc_top_sse_no_aesni.c index 15543a25b6f4995ebc9e27d77378cf6a3f55daa3..99054ce893a1fd8f00dd6bbcb750ed0d8926f2e0 100644 --- a/lib/no-aesni/zuc_top_sse_no_aesni.c +++ b/lib/no-aesni/zuc_top_sse_no_aesni.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -853,9 +853,6 @@ zuc_eia3_4_buffer_job_sse_no_aesni(const void * const pKey[NUM_SSE_BUFS], if (job_in_lane[i] == NULL) continue; - const uint32_t N = lengthInBits[i] + (2 * ZUC_WORD_BITS); - uint32_t L = ((N + 31) / ZUC_WORD_BITS) - - numKeyStr*(keyStreamLengthInBits / 32); uint32_t remainBits = lengthInBits[i] - numKeyStr*keyStreamLengthInBits; uint32_t *keyStr32 = (uint32_t *) keyStr[i]; @@ -887,7 +884,6 @@ zuc_eia3_4_buffer_job_sse_no_aesni(const void * const pKey[NUM_SSE_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - L -= (keyStreamLengthInBits / 32); /* Generate the next key stream 8 bytes or 16 bytes */ if (!remainBits) diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c deleted file mode 100644 index 2c4b638c748781701e3d591493595e8087c4a8dd..0000000000000000000000000000000000000000 --- a/lib/sse/mb_mgr_sse.c +++ /dev/null @@ -1,1302 +0,0 @@ -/******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -#include -#include -#include - -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse - -#include "ipsec-mb.h" -#include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" -#include "include/zuc_internal.h" -#include "include/snow3g.h" -#include "include/gcm.h" -#include "include/chacha20_poly1305.h" - -#include "include/save_xmms.h" -#include "include/asm.h" -#include "include/des.h" -#include "include/cpu_feature.h" -#include "include/noaesni.h" -#include "include/aesni_emu.h" -#include "include/error.h" - -IMB_JOB *submit_job_aes128_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes128_enc_x8_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_x8_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_x8_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_x8_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_x8_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_x8_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state); - - -IMB_JOB *submit_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *submit_job_aes256_ccm_auth_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *flush_job_aes256_ccm_auth_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes_cntr_sse(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_sse(IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eea3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -void aes_cmac_256_subkey_gen_sse(const void *key_exp, - void *key1, void *key2); -uint32_t hec_32_sse(const uint8_t *in); -uint64_t hec_64_sse(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_chacha20_enc_dec_sse(IMB_JOB *job); - -IMB_JOB *snow_v_sse(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_sse(IMB_JOB *job); - -void *poly1305_mac_scalar(IMB_JOB *job); - -#define SAVE_XMMS save_xmms -#define RESTORE_XMMS restore_xmms - -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_ptr -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_ptr -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_ptr -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_ptr -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_ptr -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_ptr -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse -#define SUBMIT_JOB_HMAC submit_job_hmac_sse -#define FLUSH_JOB_HMAC flush_job_hmac_sse -#define SUBMIT_JOB_HMAC_NI submit_job_hmac_ni_sse -#define FLUSH_JOB_HMAC_NI flush_job_hmac_ni_sse -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_sse -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_sse -#define SUBMIT_JOB_HMAC_SHA_224_NI submit_job_hmac_sha_224_ni_sse -#define FLUSH_JOB_HMAC_SHA_224_NI flush_job_hmac_sha_224_ni_sse -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_sse -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_sse -#define SUBMIT_JOB_HMAC_SHA_256_NI submit_job_hmac_sha_256_ni_sse -#define FLUSH_JOB_HMAC_SHA_256_NI flush_job_hmac_sha_256_ni_sse -#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse -#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse -#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse -#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse -#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse -#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse -#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse -#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse - -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_sse -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_sse -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_sse -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_sse -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_sse -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_sse -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_sse -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_sse - -#define AES_CBC_DEC_128 aes128_cbc_dec_ptr -#define AES_CBC_DEC_192 aes192_cbc_dec_ptr -#define AES_CBC_DEC_256 aes256_cbc_dec_ptr - -#define AES_CNTR_128 aes_cntr_128_sse -#define AES_CNTR_192 aes_cntr_192_sse -#define AES_CNTR_256 aes_cntr_256_sse - -#define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse -#define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse - -#define AES_ECB_ENC_128 aes_ecb_enc_128_sse -#define AES_ECB_ENC_192 aes_ecb_enc_192_sse -#define AES_ECB_ENC_256 aes_ecb_enc_256_sse -#define AES_ECB_DEC_128 aes_ecb_dec_128_sse -#define AES_ECB_DEC_192 aes_ecb_dec_192_sse -#define AES_ECB_DEC_256 aes_ecb_dec_256_sse - -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse - -#define AES_GCM_DEC_128 aes_gcm_dec_128_sse -#define AES_GCM_ENC_128 aes_gcm_enc_128_sse -#define AES_GCM_DEC_192 aes_gcm_dec_192_sse -#define AES_GCM_ENC_192 aes_gcm_enc_192_sse -#define AES_GCM_DEC_256 aes_gcm_dec_256_sse -#define AES_GCM_ENC_256 aes_gcm_enc_256_sse - -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_sse -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse - -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse - -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_sse -#define FLUSH_JOB flush_job_sse -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse -#define GET_NEXT_JOB get_next_job_sse -#define GET_COMPLETED_JOB get_completed_job_sse - -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse -#define QUEUE_SIZE queue_size_sse - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE - -/* ====================================================================== */ - -#define AES_CFB_128_ONE aes_cfb_128_one_sse -#define AES_CFB_256_ONE aes_cfb_256_one_sse - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_ptr -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_ptr - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_ptr -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_ptr - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_ptr -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_ptr - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_ptr -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_ptr - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_sse -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_sse -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_sse -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_sse - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_sse -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_sse -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_sse -#define POLY1305_MAC poly1305_mac_scalar - -#define SUBMIT_JOB_SNOW_V snow_v_sse -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_sse - -/* ====================================================================== */ - -/* - * Used to decide if SHA1/SHA256 SIMD or SHA1NI OOO scheduler should be - * called. - */ -#define HASH_USE_SHAEXT 1 - - -/* ====================================================================== */ - -uint32_t -ethernet_fcs_sse_local(const void *msg, const uint64_t len, - const void *tag_ouput); - -#define ETHERNET_FCS ethernet_fcs_sse_local - -uint32_t ethernet_fcs_sse(const void *msg, const uint64_t len); -uint32_t crc16_x25_sse(const void *msg, const uint64_t len); -uint32_t crc32_sctp_sse(const void *msg, const uint64_t len); -uint32_t crc24_lte_a_sse(const void *msg, const uint64_t len); -uint32_t crc24_lte_b_sse(const void *msg, const uint64_t len); -uint32_t crc16_fp_data_sse(const void *msg, const uint64_t len); -uint32_t crc11_fp_header_sse(const void *msg, const uint64_t len); -uint32_t crc7_fp_header_sse(const void *msg, const uint64_t len); -uint32_t crc10_iuup_data_sse(const void *msg, const uint64_t len); -uint32_t crc6_iuup_header_sse(const void *msg, const uint64_t len); -uint32_t crc32_wimax_ofdma_data_sse(const void *msg, const uint64_t len); -uint32_t crc8_wimax_ofdma_hcs_sse(const void *msg, const uint64_t len); - -/* ====================================================================== */ - -/* - * CBC decrypt function pointers - */ - -typedef void (* cbc_dec_fn_t)(const void *, const uint8_t *, const void *, - void *, uint64_t); - -static cbc_dec_fn_t aes128_cbc_dec_ptr = aes_cbc_dec_128_sse; -static cbc_dec_fn_t aes192_cbc_dec_ptr = aes_cbc_dec_192_sse; -static cbc_dec_fn_t aes256_cbc_dec_ptr = aes_cbc_dec_256_sse; - -/* - * CBC encrypt function pointers - */ - -typedef IMB_JOB *(*aes_submit_job_t)(MB_MGR_AES_OOO *, IMB_JOB *); - -static aes_submit_job_t submit_job_aes128_enc_ptr = - submit_job_aes128_enc_sse; -static aes_submit_job_t submit_job_aes192_enc_ptr = - submit_job_aes192_enc_sse; -static aes_submit_job_t submit_job_aes256_enc_ptr = - submit_job_aes256_enc_sse; - -typedef IMB_JOB *(*aes_flush_job_t)(MB_MGR_AES_OOO *); - -static aes_flush_job_t flush_job_aes128_enc_ptr = flush_job_aes128_enc_sse; -static aes_flush_job_t flush_job_aes192_enc_ptr = flush_job_aes192_enc_sse; -static aes_flush_job_t flush_job_aes256_enc_ptr = flush_job_aes256_enc_sse; - -/* ====================================================================== */ - -/* - * CMAC function pointers - */ - -typedef IMB_JOB *(*cmac_submit_job_t)(MB_MGR_CMAC_OOO *, IMB_JOB *); -typedef IMB_JOB *(*cmac_flush_job_t)(MB_MGR_CMAC_OOO *); - -static cmac_submit_job_t submit_job_aes128_cmac_auth_ptr = - submit_job_aes128_cmac_auth_sse; -static cmac_flush_job_t flush_job_aes128_cmac_auth_ptr = - flush_job_aes128_cmac_auth_sse; -static cmac_submit_job_t submit_job_aes256_cmac_auth_ptr = - submit_job_aes256_cmac_auth_sse; -static cmac_flush_job_t flush_job_aes256_cmac_auth_ptr = - flush_job_aes256_cmac_auth_sse; - -/* ====================================================================== */ - -/* - * CCM function pointers - */ - -typedef IMB_JOB *(*ccm_submit_job_t)(MB_MGR_CCM_OOO *, IMB_JOB *); -typedef IMB_JOB *(*ccm_flush_job_t)(MB_MGR_CCM_OOO *); - -static ccm_submit_job_t submit_job_aes128_ccm_auth_ptr = - submit_job_aes128_ccm_auth_sse; -static ccm_flush_job_t flush_job_aes128_ccm_auth_ptr = - flush_job_aes128_ccm_auth_sse; - -static ccm_submit_job_t submit_job_aes256_ccm_auth_ptr = - submit_job_aes256_ccm_auth_sse; -static ccm_flush_job_t flush_job_aes256_ccm_auth_ptr = - flush_job_aes256_ccm_auth_sse; - -/* ====================================================================== */ - -/* - * GCM submit / flush API for SSE arch - */ -static IMB_JOB * -submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_bit_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -/* ====================================================================== */ - -static IMB_JOB * -(*submit_job_zuc_eea3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eea3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc_eea3_sse) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eea3_no_gfni_sse; - -static IMB_JOB * -(*submit_job_zuc_eia3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eia3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc_eia3_sse) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eia3_no_gfni_sse; - -static IMB_JOB * -(*submit_job_zuc256_eea3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc256_eea3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc256_eea3_sse) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc256_eea3_no_gfni_sse; - -static IMB_JOB * -(*submit_job_zuc256_eia3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz) = - submit_job_zuc256_eia3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc256_eia3_sse) - (MB_MGR_ZUC_OOO *state, const uint64_t tag_sz) = - flush_job_zuc256_eia3_no_gfni_sse; - -static void -reset_ooo_mgrs(IMB_MGR *state) -{ - unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - - /* Init AES out-of-order fields */ - memset(aes128_ooo->lens, 0xFF, sizeof(aes128_ooo->lens)); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) { - aes128_ooo->unused_lanes = 0xF76543210; - submit_job_aes128_enc_ptr = submit_job_aes128_enc_x8_sse; - flush_job_aes128_enc_ptr = flush_job_aes128_enc_x8_sse; - } else { - aes128_ooo->unused_lanes = 0xF3210; - } - - memset(aes192_ooo->lens, 0xFF, sizeof(aes192_ooo->lens)); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) { - aes192_ooo->unused_lanes = 0xF76543210; - submit_job_aes192_enc_ptr = submit_job_aes192_enc_x8_sse; - flush_job_aes192_enc_ptr = flush_job_aes192_enc_x8_sse; - } else { - aes192_ooo->unused_lanes = 0xF3210; - } - - memset(aes256_ooo->lens, 0xFF, sizeof(aes256_ooo->lens)); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) { - aes256_ooo->unused_lanes = 0xF76543210; - submit_job_aes256_enc_ptr = submit_job_aes256_enc_x8_sse; - flush_job_aes256_enc_ptr = flush_job_aes256_enc_x8_sse; - } else { - aes256_ooo->unused_lanes = 0xF3210; - } - - if (state->features & IMB_FEATURE_GFNI) { - /* change AES-CBC decrypt implementation */ - aes128_cbc_dec_ptr = aes_cbc_dec_128_by8_sse; - aes192_cbc_dec_ptr = aes_cbc_dec_192_by8_sse; - aes256_cbc_dec_ptr = aes_cbc_dec_256_by8_sse; - } - - /* DOCSIS SEC BPI uses same settings as AES CBC */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis128_sec_ooo->unused_lanes = 0xF76543210; - else - docsis128_sec_ooo->unused_lanes = 0xF3210; - - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - else - docsis128_crc32_sec_ooo->unused_lanes = 0xF3210; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis256_sec_ooo->unused_lanes = 0xF76543210; - else - docsis256_sec_ooo->unused_lanes = 0xF3210; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - else - docsis256_crc32_sec_ooo->unused_lanes = 0xF3210; - - /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFF03020100; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFF03020100; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFF03020100; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; - - /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0xFFFF; - hmac_sha_1_ooo->lens[5] = 0xFFFF; - hmac_sha_1_ooo->lens[6] = 0xFFFF; - hmac_sha_1_ooo->lens[7] = 0xFFFF; - hmac_sha_1_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5*4] = 0x80; - p[64-2] = 0x02; - p[64-1] = 0xA0; - } - -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - /* Init HMAC/SHA1 NI out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0xFFFF; - hmac_sha_1_ooo->lens[3] = 0xFFFF; - hmac_sha_1_ooo->lens[4] = 0xFFFF; - hmac_sha_1_ooo->lens[5] = 0xFFFF; - hmac_sha_1_ooo->lens[6] = 0xFFFF; - hmac_sha_1_ooo->lens[7] = 0xFFFF; - hmac_sha_1_ooo->unused_lanes = 0xFF0100; - } -#endif /* HASH_USE_SHAEXT */ - - /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0xFFFF; - hmac_sha_224_ooo->lens[5] = 0xFFFF; - hmac_sha_224_ooo->lens[6] = 0xFFFF; - hmac_sha_224_ooo->lens[7] = 0xFFFF; - hmac_sha_224_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7*4] = 0x80; /* digest 7 words long */ - p[64-2] = 0x02; /* length in little endian = 0x02E0 */ - p[64-1] = 0xE0; - } -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - /* Init HMAC/SHA224 NI out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0xFFFF; - hmac_sha_224_ooo->lens[3] = 0xFFFF; - hmac_sha_224_ooo->lens[4] = 0xFFFF; - hmac_sha_224_ooo->lens[5] = 0xFFFF; - hmac_sha_224_ooo->lens[6] = 0xFFFF; - hmac_sha_224_ooo->lens[7] = 0xFFFF; - hmac_sha_224_ooo->unused_lanes = 0xFF0100; - } -#endif /* HASH_USE_SHAEXT */ - - /* Init HMAC/SHA_256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0xFFFF; - hmac_sha_256_ooo->lens[5] = 0xFFFF; - hmac_sha_256_ooo->lens[6] = 0xFFFF; - hmac_sha_256_ooo->lens[7] = 0xFFFF; - hmac_sha_256_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); /* digest is 8*4 bytes long */ - p[8*4] = 0x80; - p[64-2] = 0x03; /* length of (opad (64*8) bits + 256 bits) - * in hex is 0x300 */ - p[64-1] = 0x00; - } -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - /* Init HMAC/SHA256 NI out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0xFFFF; - hmac_sha_256_ooo->lens[3] = 0xFFFF; - hmac_sha_256_ooo->lens[4] = 0xFFFF; - hmac_sha_256_ooo->lens[5] = 0xFFFF; - hmac_sha_256_ooo->lens[6] = 0xFFFF; - hmac_sha_256_ooo->lens[7] = 0xFFFF; - hmac_sha_256_ooo->unused_lanes = 0xFF0100; - } -#endif /* HASH_USE_SHAEXT */ - - /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0xFFFF; - hmac_sha_384_ooo->lens[3] = 0xFFFF; - hmac_sha_384_ooo->lens[4] = 0xFFFF; - hmac_sha_384_ooo->lens[5] = 0xFFFF; - hmac_sha_384_ooo->lens[6] = 0xFFFF; - hmac_sha_384_ooo->lens[7] = 0xFFFF; - hmac_sha_384_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < SSE_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */ - /* - * hmac outer block length always of fixed size, it is OKey - * length, a whole message block length, 1024 bits, with padding - * plus the length of the inner digest, which is 384 bits - * 1408 bits == 0x0580. The input message block needs to be - * converted to big endian within the sha implementation - * before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } - - /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0xFFFF; - hmac_sha_512_ooo->lens[3] = 0xFFFF; - hmac_sha_512_ooo->lens[4] = 0xFFFF; - hmac_sha_512_ooo->lens[5] = 0xFFFF; - hmac_sha_512_ooo->lens[6] = 0xFFFF; - hmac_sha_512_ooo->lens[7] = 0xFFFF; - hmac_sha_512_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < SSE_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */ - /* - * hmac outer block length always of fixed size, it is OKey - * length, a whole message block length, 1024 bits, with padding - * plus the length of the inner digest, which is 512 bits - * 1536 bits == 0x600. The input message block needs to be - * converted to big endian within the sha implementation - * before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } - - /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0xFFFF; - hmac_md5_ooo->lens[9] = 0xFFFF; - hmac_md5_ooo->lens[10] = 0xFFFF; - hmac_md5_ooo->lens[11] = 0xFFFF; - hmac_md5_ooo->lens[12] = 0xFFFF; - hmac_md5_ooo->lens[13] = 0xFFFF; - hmac_md5_ooo->lens[14] = 0xFFFF; - hmac_md5_ooo->lens[15] = 0xFFFF; - hmac_md5_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < SSE_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4*4] = 0x80; - p[64-7] = 0x02; - p[64-8] = 0x80; - } - - /* Init AES/XCBC OOO fields */ - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < 4; j++) { - aes_xcbc_ooo->lens[j] = 0xFFFF; - aes_xcbc_ooo->ldata[j].job_in_lane = NULL; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15); - } - aes_xcbc_ooo->num_lanes_inuse = 0; - - /* Init AES-CCM auth out-of-order fields */ - memset(aes_ccm_ooo->init_done, 0, - sizeof(aes_ccm_ooo->init_done)); - memset(aes_ccm_ooo->lens, 0xff, sizeof(aes_ccm_ooo->lens)); - memset(aes_ccm_ooo->job_in_lane, 0, - sizeof(aes_ccm_ooo->job_in_lane)); - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes128_ccm_auth_ptr = - submit_job_aes128_ccm_auth_x8_sse; - flush_job_aes128_ccm_auth_ptr = - flush_job_aes128_ccm_auth_x8_sse; - aes_ccm_ooo->unused_lanes = 0xF76543210; - } else { - aes_ccm_ooo->unused_lanes = 0xF3210; - } - aes_ccm_ooo->num_lanes_inuse = 0; - - memset(aes256_ccm_ooo->init_done, 0, - sizeof(aes256_ccm_ooo->init_done)); - memset(aes256_ccm_ooo->lens, 0xff, sizeof(aes256_ccm_ooo->lens)); - memset(aes256_ccm_ooo->job_in_lane, 0, - sizeof(aes256_ccm_ooo->job_in_lane)); - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes256_ccm_auth_ptr = - submit_job_aes256_ccm_auth_x8_sse; - flush_job_aes256_ccm_auth_ptr = - flush_job_aes256_ccm_auth_x8_sse; - aes256_ccm_ooo->unused_lanes = 0xF76543210; - } else { - aes256_ccm_ooo->unused_lanes = 0xF3210; - } - aes256_ccm_ooo->num_lanes_inuse = 0; - - /* Init AES-CMAC auth out-of-order fields */ - memset(aes_cmac_ooo->init_done, 0, - sizeof(aes_cmac_ooo->init_done)); - memset(aes_cmac_ooo->lens, 0xff, - sizeof(aes_cmac_ooo->lens)); - memset(aes_cmac_ooo->job_in_lane, 0, - sizeof(aes_cmac_ooo->job_in_lane)); - aes_cmac_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes128_cmac_auth_ptr = - submit_job_aes128_cmac_auth_x8_sse; - flush_job_aes128_cmac_auth_ptr = - flush_job_aes128_cmac_auth_x8_sse; - aes_cmac_ooo->unused_lanes = 0xF76543210; - } else { - aes_cmac_ooo->unused_lanes = 0xF3210; - } - - memset(aes256_cmac_ooo->init_done, 0, - sizeof(aes256_cmac_ooo->init_done)); - memset(aes256_cmac_ooo->lens, 0xff, - sizeof(aes256_cmac_ooo->lens)); - memset(aes256_cmac_ooo->job_in_lane, 0, - sizeof(aes256_cmac_ooo->job_in_lane)); - aes256_cmac_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes256_cmac_auth_ptr = - submit_job_aes256_cmac_auth_x8_sse; - flush_job_aes256_cmac_auth_ptr = - flush_job_aes256_cmac_auth_x8_sse; - aes256_cmac_ooo->unused_lanes = 0xF76543210; - } else { - aes256_cmac_ooo->unused_lanes = 0xF3210; - } - - /* Init AES-CBCS out-of-order fields */ - memset(aes128_cbcs_ooo->lens, 0xFF, sizeof(aes128_cbcs_ooo->lens)); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->num_lanes_inuse = 0; - aes128_cbcs_ooo->unused_lanes = 0xF3210; -} - -IMB_DLL_LOCAL void -init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) -{ -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_SSE; - - if (state->features & IMB_FEATURE_GFNI) { - submit_job_zuc_eea3_sse = submit_job_zuc_eea3_gfni_sse; - flush_job_zuc_eea3_sse = flush_job_zuc_eea3_gfni_sse; - submit_job_zuc_eia3_sse = submit_job_zuc_eia3_gfni_sse; - flush_job_zuc_eia3_sse = flush_job_zuc_eia3_gfni_sse; - submit_job_zuc256_eea3_sse = submit_job_zuc256_eea3_gfni_sse; - flush_job_zuc256_eea3_sse = flush_job_zuc256_eea3_gfni_sse; - submit_job_zuc256_eia3_sse = submit_job_zuc256_eia3_gfni_sse; - flush_job_zuc256_eia3_sse = flush_job_zuc256_eia3_gfni_sse; - } - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set SSE handlers */ - state->get_next_job = get_next_job_sse; - state->submit_job = submit_job_sse; - state->submit_job_nocheck = submit_job_nocheck_sse; - state->get_completed_job = get_completed_job_sse; - state->flush_job = flush_job_sse; - state->queue_size = queue_size_sse; - state->keyexp_128 = aes_keyexp_128_sse; - state->keyexp_192 = aes_keyexp_192_sse; - state->keyexp_256 = aes_keyexp_256_sse; - state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse; - state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_sse; - state->xcbc_keyexp = aes_xcbc_expand_key_sse; - state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_sse; - state->sha1 = sha1_sse; - state->sha224_one_block = sha224_one_block_sse; - state->sha224 = sha224_sse; - state->sha256_one_block = sha256_one_block_sse; - state->sha256 = sha256_sse; - state->sha384_one_block = sha384_one_block_sse; - state->sha384 = sha384_sse; - state->sha512_one_block = sha512_one_block_sse; - state->sha512 = sha512_sse; - state->md5_one_block = md5_one_block_sse; - state->aes128_cfb_one = aes_cfb_128_one_sse; - state->crc32_ethernet_fcs = ethernet_fcs_sse; - state->crc16_x25 = crc16_x25_sse; - state->crc32_sctp = crc32_sctp_sse; - state->crc24_lte_a = crc24_lte_a_sse; - state->crc24_lte_b = crc24_lte_b_sse; - state->crc16_fp_data = crc16_fp_data_sse; - state->crc11_fp_header = crc11_fp_header_sse; - state->crc7_fp_header = crc7_fp_header_sse; - state->crc10_iuup_data = crc10_iuup_data_sse; - state->crc6_iuup_header = crc6_iuup_header_sse; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_sse; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_sse; - - state->eea3_1_buffer = zuc_eea3_1_buffer_sse; - if (state->features & IMB_FEATURE_GFNI) { - state->eea3_4_buffer = zuc_eea3_4_buffer_gfni_sse; - state->eea3_n_buffer = zuc_eea3_n_buffer_gfni_sse; - state->eia3_n_buffer = zuc_eia3_n_buffer_gfni_sse; - } else { - state->eea3_4_buffer = zuc_eea3_4_buffer_sse; - state->eea3_n_buffer = zuc_eea3_n_buffer_sse; - state->eia3_n_buffer = zuc_eia3_n_buffer_sse; - } - - state->eia3_1_buffer = zuc_eia3_1_buffer_sse; - - state->f8_1_buffer = kasumi_f8_1_buffer_sse; - state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse; - state->f8_2_buffer = kasumi_f8_2_buffer_sse; - state->f8_3_buffer = kasumi_f8_3_buffer_sse; - state->f8_4_buffer = kasumi_f8_4_buffer_sse; - state->f8_n_buffer = kasumi_f8_n_buffer_sse; - state->f9_1_buffer = kasumi_f9_1_buffer_sse; - state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse; - state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse; - state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse; - state->kasumi_key_sched_size = kasumi_key_sched_size_sse; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_sse; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_sse; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse; - state->snow3g_init_key_sched = snow3g_init_key_sched_sse; - state->snow3g_key_sched_size = snow3g_key_sched_size_sse; - - state->gcm128_enc = aes_gcm_enc_128_sse; - state->gcm192_enc = aes_gcm_enc_192_sse; - state->gcm256_enc = aes_gcm_enc_256_sse; - state->gcm128_dec = aes_gcm_dec_128_sse; - state->gcm192_dec = aes_gcm_dec_192_sse; - state->gcm256_dec = aes_gcm_dec_256_sse; - state->gcm128_init = aes_gcm_init_128_sse; - state->gcm192_init = aes_gcm_init_192_sse; - state->gcm256_init = aes_gcm_init_256_sse; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_sse; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_sse; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_sse; - state->gcm128_enc_update = aes_gcm_enc_128_update_sse; - state->gcm192_enc_update = aes_gcm_enc_192_update_sse; - state->gcm256_enc_update = aes_gcm_enc_256_update_sse; - state->gcm128_dec_update = aes_gcm_dec_128_update_sse; - state->gcm192_dec_update = aes_gcm_dec_192_update_sse; - state->gcm256_dec_update = aes_gcm_dec_256_update_sse; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse; - state->gcm128_precomp = aes_gcm_precomp_128_sse; - state->gcm192_precomp = aes_gcm_precomp_192_sse; - state->gcm256_precomp = aes_gcm_precomp_256_sse; - state->gcm128_pre = aes_gcm_pre_128_sse; - state->gcm192_pre = aes_gcm_pre_192_sse; - state->gcm256_pre = aes_gcm_pre_256_sse; - state->ghash = ghash_sse; - state->ghash_pre = ghash_pre_sse; - - state->gmac128_init = imb_aes_gmac_init_128_sse; - state->gmac192_init = imb_aes_gmac_init_192_sse; - state->gmac256_init = imb_aes_gmac_init_256_sse; - state->gmac128_update = imb_aes_gmac_update_128_sse; - state->gmac192_update = imb_aes_gmac_update_192_sse; - state->gmac256_update = imb_aes_gmac_update_256_sse; - state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; - state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; - state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; - - state->hec_32 = hec_32_sse; - state->hec_64 = hec_64_sse; - - state->chacha20_poly1305_init = init_chacha20_poly1305_sse; - state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_sse; - state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_sse; - state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_sse; -} - -void -init_mb_mgr_sse(IMB_MGR *state) -{ - init_mb_mgr_sse_internal(state, 1); -} - -#include "mb_mgr_code.h" diff --git a/lib/sse_t1/aes128_cbc_dec_by4_sse.asm b/lib/sse_t1/aes128_cbc_dec_by4_sse.asm index 276127df4154fed85d9941ea33a63ddeb4ad069e..186d1be12434a7452d255dbe7c8baedcc1ee9575 100644 --- a/lib/sse_t1/aes128_cbc_dec_by4_sse.asm +++ b/lib/sse_t1/aes128_cbc_dec_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -41,8 +41,8 @@ ; arg 4: OUT: pointer to output (plain text) ; arg 5: LEN: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_DEC_128 %define AES_CBC_DEC_128 aes_cbc_dec_128_sse diff --git a/lib/sse_t1/aes128_cbc_enc_x4_sse.asm b/lib/sse_t1/aes128_cbc_enc_x4_sse.asm index a2713c2c537552d3f2d6660964aa554679a0c02d..8850b66b5be2e344a3bccf124cd996ebccde8e7e 100644 --- a/lib/sse_t1/aes128_cbc_enc_x4_sse.asm +++ b/lib/sse_t1/aes128_cbc_enc_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,9 +29,9 @@ ;;; processes 4 buffers at a time, single data structure as input ;;; Updates In and Out pointers at end -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" %define MOVDQ movdqu ;; assume buffers not aligned %macro pxor2 2 diff --git a/lib/sse_t1/aes128_cbc_mac_x4_sse.asm b/lib/sse_t1/aes128_cbc_mac_x4_sse.asm index 52726f1b72beea68600b035a9c7d86340141f4f2..4d0e91822e384d22bf822f3bd39ddb623cd5204d 100644 --- a/lib/sse_t1/aes128_cbc_mac_x4_sse.asm +++ b/lib/sse_t1/aes128_cbc_mac_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes128_cbcs_1_9_dec_by4_sse.asm b/lib/sse_t1/aes128_cbcs_1_9_dec_by4_sse.asm index fcef251c8945680d1bb5889e951457e538e6ecb1..e1deeb92f0037d9231da13fec5fd5180d6175a00 100644 --- a/lib/sse_t1/aes128_cbcs_1_9_dec_by4_sse.asm +++ b/lib/sse_t1/aes128_cbcs_1_9_dec_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes128_cbcs_1_9_enc_x4_sse.asm b/lib/sse_t1/aes128_cbcs_1_9_enc_x4_sse.asm index 77e9a3a0d884fd42e38ec2db0995a904cab8da7e..93fc66ed9d98664abcf35e117937d1a69f6f7bf6 100644 --- a/lib/sse_t1/aes128_cbcs_1_9_enc_x4_sse.asm +++ b/lib/sse_t1/aes128_cbcs_1_9_enc_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes128_cntr_by8_sse.asm b/lib/sse_t1/aes128_cntr_by8_sse.asm index 7005d5639ab89debb8c98a1b148acd582142561e..84879607cbb0eab6c9532bb348aaeb7e3c0007e0 100644 --- a/lib/sse_t1/aes128_cntr_by8_sse.asm +++ b/lib/sse_t1/aes128_cntr_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ; routine to do AES128 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level diff --git a/lib/sse_t1/aes128_cntr_ccm_by8_sse.asm b/lib/sse_t1/aes128_cntr_ccm_by8_sse.asm index 3b2e16c8d01503f86bf0ab811f1f630813ec2593..48d3f067631487704a4291ace42ca5319282fc39 100644 --- a/lib/sse_t1/aes128_cntr_ccm_by8_sse.asm +++ b/lib/sse_t1/aes128_cntr_ccm_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes128_ecb_by4_sse.asm b/lib/sse_t1/aes128_ecb_by4_sse.asm index f1fafa2709a6f2c78d6b8fb80001d60b90253b32..2bc4dbdfd308462afe979d505643dbf08ccb1c87 100644 --- a/lib/sse_t1/aes128_ecb_by4_sse.asm +++ b/lib/sse_t1/aes128_ecb_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -42,8 +42,8 @@ ; arg 4: LEN: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef AES_ECB_ENC_256 %ifndef AES_ECB_ENC_192 diff --git a/lib/sse_t1/aes128_xcbc_mac_x4_sse.asm b/lib/sse_t1/aes128_xcbc_mac_x4_sse.asm index 376327c31e14a818363948bcb3a931e4f417205b..cc142548312472402253bd74de1fbe8aaf490987 100644 --- a/lib/sse_t1/aes128_xcbc_mac_x4_sse.asm +++ b/lib/sse_t1/aes128_xcbc_mac_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes192_cbc_dec_by4_sse.asm b/lib/sse_t1/aes192_cbc_dec_by4_sse.asm index 9e7385f6b2bee56c7ee41fe26e46fe183e4788d5..af8830544c0da008ca7a6e7e69c8f3146ce7cf1e 100644 --- a/lib/sse_t1/aes192_cbc_dec_by4_sse.asm +++ b/lib/sse_t1/aes192_cbc_dec_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -41,8 +41,8 @@ ; arg 4: OUT: pointer to output (plain text) ; arg 5: LEN: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_DEC_192 %define AES_CBC_DEC_192 aes_cbc_dec_192_sse diff --git a/lib/sse_t1/aes192_cbc_enc_x4_sse.asm b/lib/sse_t1/aes192_cbc_enc_x4_sse.asm index 9ed053e04d38cf8b00555a3160de3c85a6878f00..f86494d698c5f95aadbdf6c20a6a3e11f77049cb 100644 --- a/lib/sse_t1/aes192_cbc_enc_x4_sse.asm +++ b/lib/sse_t1/aes192_cbc_enc_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,9 +29,9 @@ ;;; process 4 buffers at a time, single data structure as input ;;; Updates In and Out pointers at end -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" %define MOVDQ movdqu ;; assume buffers not aligned %macro pxor2 2 diff --git a/lib/sse_t1/aes192_cntr_by8_sse.asm b/lib/sse_t1/aes192_cntr_by8_sse.asm index d0a861dd133cffe0001fdf893ce5e1ee68ebcfa4..2abfbbb170b855153e46ad4bb15b92c130d237b6 100644 --- a/lib/sse_t1/aes192_cntr_by8_sse.asm +++ b/lib/sse_t1/aes192_cntr_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ; routine to do AES192 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level diff --git a/lib/sse_t1/aes192_ecb_by4_sse.asm b/lib/sse_t1/aes192_ecb_by4_sse.asm index fcad26e596cfc2b6423d5a2f2659956d66e959ec..8a607f1276d0078471ac1730093949dd745107f3 100644 --- a/lib/sse_t1/aes192_ecb_by4_sse.asm +++ b/lib/sse_t1/aes192_ecb_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes256_cbc_dec_by4_sse.asm b/lib/sse_t1/aes256_cbc_dec_by4_sse.asm index e8bb697512913bd9721abab5b1fcef18c1f512c8..40e29749d2fa6b64c0c6b02b162946c057f57f21 100644 --- a/lib/sse_t1/aes256_cbc_dec_by4_sse.asm +++ b/lib/sse_t1/aes256_cbc_dec_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -42,8 +42,8 @@ ; arg 5: sp: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_DEC_256 %define AES_CBC_DEC_256 aes_cbc_dec_256_sse diff --git a/lib/sse_t1/aes256_cbc_enc_x4_sse.asm b/lib/sse_t1/aes256_cbc_enc_x4_sse.asm index 66f64efe4971285bfdd46f170157ac6181ecbb57..b7f23a8a619b048a032deaf3335080456aebcc23 100644 --- a/lib/sse_t1/aes256_cbc_enc_x4_sse.asm +++ b/lib/sse_t1/aes256_cbc_enc_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,9 +29,9 @@ ;;; process 4 buffers at a time, single data structure as input ;;; Updates In and Out pointers at end -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" %define MOVDQ movdqu ;; assume buffers not aligned %macro pxor2 2 diff --git a/lib/sse_t1/aes256_cbc_mac_x4_sse.asm b/lib/sse_t1/aes256_cbc_mac_x4_sse.asm index 998c8434eaea3b34ce0e4d291d7df080923ba786..77a9631bdeb3ce4d1500f5927e27fa6adf83293c 100644 --- a/lib/sse_t1/aes256_cbc_mac_x4_sse.asm +++ b/lib/sse_t1/aes256_cbc_mac_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes256_cntr_by8_sse.asm b/lib/sse_t1/aes256_cntr_by8_sse.asm index c9ca98770fd5b68d4afdadddbf676586026c3fc8..f3e304bb1deea6913865a496be1f93a1a41a6e0b 100644 --- a/lib/sse_t1/aes256_cntr_by8_sse.asm +++ b/lib/sse_t1/aes256_cntr_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" %include "include/const.inc" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" ; routine to do AES256 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level diff --git a/lib/sse_t1/aes256_cntr_ccm_by8_sse.asm b/lib/sse_t1/aes256_cntr_ccm_by8_sse.asm index 94c9c2d265e9d1c6a40c05e4f4e977d070005177..6129b1a14027032bf5d41149c67782c9323b5ef3 100644 --- a/lib/sse_t1/aes256_cntr_ccm_by8_sse.asm +++ b/lib/sse_t1/aes256_cntr_ccm_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes256_ecb_by4_sse.asm b/lib/sse_t1/aes256_ecb_by4_sse.asm index 703ede7355cf433c303642032960bc08ffd53982..cb063e85d65a2f0e63d2a9c7312b05573b60c447 100644 --- a/lib/sse_t1/aes256_ecb_by4_sse.asm +++ b/lib/sse_t1/aes256_ecb_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/aes_cfb_sse.asm b/lib/sse_t1/aes_cfb_sse.asm index 765ef5428adbee6ff22c77818536f6053f622ec9..4df7df6551d2dd2271a195e53f3f361184c7aff9 100644 --- a/lib/sse_t1/aes_cfb_sse.asm +++ b/lib/sse_t1/aes_cfb_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" ;;; Routines to do 128/256 bit CFB AES encrypt/decrypt operations on one block only. @@ -206,6 +206,7 @@ AES_CFB_128_ONE: MKGLOBAL(AES_CFB_256_ONE,function,) align 32 AES_CFB_256_ONE: + endbranch64 do_cfb 13 ret diff --git a/lib/sse_t1/aes_ecb_quic_x8_sse.asm b/lib/sse_t1/aes_ecb_quic_x8_sse.asm new file mode 100644 index 0000000000000000000000000000000000000000..323aa4fcb3b09b4372f67204fe47f72ab6a51a1f --- /dev/null +++ b/lib/sse_t1/aes_ecb_quic_x8_sse.asm @@ -0,0 +1,246 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB encrypt for QUIC on 16-byte buffers + +; XMM registers are clobbered. Saving/restoring must be done at a higher level + +; void aes_ecb_quic_enc_x_sse(void *in, +; UINT128 keys[], +; void *out, +; UINT64 num_buffers); +; +; x = key size (128/256) +; arg 1: IN: array of pointers to input buffers +; arg 2: KEYS: pointer to keys (common for all buffers) +; arg 3: OUT: array of pointers to output buffers) +; arg 4: N_BUFS: number of 16-byte buffers +; + +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" +%include "include/cet.inc" + +%ifndef AES_ECB_QUIC_ENC_128 +%define AES_ECB_QUIC_ENC_128 aes_ecb_quic_enc_128_sse +%define AES_ECB_QUIC_ENC_256 aes_ecb_quic_enc_256_sse +%endif + +;; ============================================================================= +;; Loads 1 AES block from up to 8 buffers into XMM registers +%macro XMM_LOAD_BLOCKS_MULT_IN_0_8 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%ARRAY_INP %2 ; [in] array of input data pointers +%define %%INP %3 ; [clobbered] input data pointer to read 16 bytes from +%define %%DST0 %4 ; [out] XMM register with loaded data +%define %%DST1 %5 ; [out] XMM register with loaded data +%define %%DST2 %6 ; [out] XMM register with loaded data +%define %%DST3 %7 ; [out] XMM register with loaded data +%define %%DST4 %8 ; [out] XMM register with loaded data +%define %%DST5 %9 ; [out] XMM register with loaded data +%define %%DST6 %10 ; [out] XMM register with loaded data +%define %%DST7 %11 ; [out] XMM register with loaded data + +%assign dst_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%DSTREG %%DST %+ dst_idx + mov %%INP, [%%ARRAY_INP + dst_idx*8] + movdqu %%DSTREG, [%%INP] +%undef %%DSTREG +%assign dst_idx (dst_idx + 1) +%endrep + +%endmacro + +;; ============================================================================= +;; Stores 1 AES blocks to up to 8 buffers from XMM registers +%macro XMM_STORE_BLOCKS_MULT_OUT_0_8 12 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 8) +%define %%ARRAY_OUTP %2 ; [in] array of output data pointers to write to +%define %%OUTP %3 ; [clobbered] output data pointer to write to +%define %%SRC0 %4 ; [in] XMM register with data to store +%define %%SRC1 %5 ; [in] XMM register with data to store +%define %%SRC2 %6 ; [in] XMM register with data to store +%define %%SRC3 %7 ; [in] XMM register with data to store +%define %%SRC4 %8 ; [in] XMM register with data to store +%define %%SRC5 %9 ; [in] XMM register with data to store +%define %%SRC6 %10 ; [in] XMM register with data to store +%define %%SRC7 %11 ; [in] XMM register with data to store +%define %%TMP %12 ; [clobbered] Temporary GP register + +%assign src_idx 0 +%rep (%%NUM_BLOCKS) +%xdefine %%SRCREG %%SRC %+ src_idx + ; Store 5 bytes to each buffer + mov %%OUTP, [%%ARRAY_OUTP + src_idx*8] + movd [%%OUTP], %%SRCREG + pextrb [%%OUTP + 4], %%SRCREG, 4 +%undef %%SRCREG +%assign src_idx (src_idx + 1) +%endrep + +%endmacro + +%ifdef LINUX +%define IN rdi +%define KEYS rsi +%define OUT rdx +%define N_BUFS rcx +%else +%define IN rcx +%define KEYS rdx +%define OUT r8 +%define N_BUFS r9 +%endif + +%define IDX rax +%define TMP IDX +%define XDATA0 xmm0 +%define XDATA1 xmm1 +%define XDATA2 xmm2 +%define XDATA3 xmm3 +%define XKEY0 xmm4 +%define XKEY2 xmm5 +%define XKEY4 xmm6 +%define XKEY6 xmm7 +%define XKEY10 xmm8 +%define XKEY_A xmm14 +%define XKEY_B xmm15 + +mksection .text + +%macro AES_ECB_QUIC 1 +%define %%NROUNDS %1 ; [in] Number of rounds + +%define IDX rax +%define TMP r11 +%define TMP2 r10 +%define XDATA0 xmm0 +%define XDATA1 xmm1 +%define XDATA2 xmm2 +%define XDATA3 xmm3 +%define XDATA4 xmm4 +%define XDATA5 xmm5 +%define XDATA6 xmm6 +%define XDATA7 xmm7 +%define XKEY1 xmm8 + + or N_BUFS, N_BUFS + jz %%done + xor IDX, IDX + mov TMP, N_BUFS + and TMP, 0x7 ; number of initial buffers (0 to 7 buffers) + jz %%main_loop + ; branch to different code block based on remainder + cmp TMP, 4 + je %%initial_num_buffers_is_4 + jb %%initial_num_buffers_is_3_1 + cmp TMP, 6 + je %%initial_num_buffers_is_6 + jb %%initial_num_buffers_is_5 + ja %%initial_num_buffers_is_7 +%%initial_num_buffers_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2 + ja %%initial_num_buffers_is_3 + je %%initial_num_buffers_is_2 + ;; fall through for `jmp %%initial_num_buffers_is_1` +%assign num_buffers 1 +%rep 7 +%%initial_num_buffers_is_ %+ num_buffers : + ; load initial blocks + XMM_LOAD_BLOCKS_MULT_IN_0_8 num_buffers, IN, TMP, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform aesenc encryption on initial blocks +%rep (%%NROUNDS + 1) ; 10/14 + movdqu XKEY1, [KEYS + %%I*16] + XMM_AESENC_ROUND_BLOCKS_SSE_0_8 XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, num_buffers, (%%NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store initial blocks + XMM_STORE_BLOCKS_MULT_OUT_0_8 num_buffers, OUT, TMP, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7, TMP2 + add IDX, num_buffers + cmp IDX, N_BUFS + je %%done +%assign num_buffers (num_buffers + 1) + jmp %%main_loop +%endrep +align 16 +%%main_loop: + ; load next 8 blocks + XMM_LOAD_BLOCKS_MULT_IN_0_8 8, {IN + IDX*8}, TMP, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform AES encryption on 8 blocks +%rep (%%NROUNDS + 1) ; 10/14 + movdqu XKEY1, [KEYS + %%I*16] + XMM_AESENC_ROUND_BLOCKS_SSE_0_8 XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, 8, (%%NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store 8 blocks + XMM_STORE_BLOCKS_MULT_OUT_0_8 8, {OUT + IDX*8}, TMP, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7, TMP2 + add IDX, 8 + cmp IDX, N_BUFS + jne %%main_loop +%%done: +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif + ret +%endmacro + +%ifdef AES_ECB_QUIC_ENC_128 +align 16 +MKGLOBAL(AES_ECB_QUIC_ENC_128,function,internal) +AES_ECB_QUIC_ENC_128: + endbranch64 + AES_ECB_QUIC 10 + +%endif + +%ifdef AES_ECB_QUIC_ENC_256 +align 16 +MKGLOBAL(AES_ECB_QUIC_ENC_256,function,internal) +AES_ECB_QUIC_ENC_256: + endbranch64 + AES_ECB_QUIC 14 + +%endif +mksection stack-noexec diff --git a/lib/sse_t1/chacha20_sse.asm b/lib/sse_t1/chacha20_sse.asm index 253b17dc021fd58b0c226fb43e6796631570ec70..4458a8f886f833d7459fabcabbe1187acebec65a 100644 --- a/lib/sse_t1/chacha20_sse.asm +++ b/lib/sse_t1/chacha20_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" -%include "include/chacha_poly_defines.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" +%include "include/chacha_poly_defines.inc" mksection .rodata default rel diff --git a/lib/sse_t1/crc16_x25_sse.asm b/lib/sse_t1/crc16_x25_sse.asm index 29c56b46f6cdffd9684edd2e7e75c09330ad21f4..446701f2ab5345d024cf28ea658b2dbf3c38a314 100644 --- a/lib/sse_t1/crc16_x25_sse.asm +++ b/lib/sse_t1/crc16_x25_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_refl_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/crc32_by8_sse.asm b/lib/sse_t1/crc32_by8_sse.asm index 3c4dc4012291f26e4b8085d7bcb849702a97ea79..9896f880cacea6b92f65bd3c74c32082585f61d3 100644 --- a/lib/sse_t1/crc32_by8_sse.asm +++ b/lib/sse_t1/crc32_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -35,11 +35,11 @@ ;; "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" ;; URL: http://download.intel.com/design/intarch/papers/323102.pdf -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/reg_sizes.inc" %include "include/crc32.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifndef CRC32_FN %define CRC32_FN crc32_by8_sse diff --git a/lib/sse_t1/crc32_fp_sse.asm b/lib/sse_t1/crc32_fp_sse.asm index fc10c5be1e212b3c463dc0e30757c5f0d9ccfdba..6357916a6f81731ca3f0d5d9a0494a23505c9d2a 100644 --- a/lib/sse_t1/crc32_fp_sse.asm +++ b/lib/sse_t1/crc32_fp_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/crc32_iuup_sse.asm b/lib/sse_t1/crc32_iuup_sse.asm index 2ef9bf65a8765ddeead553c218faf09649d61e99..e08c83d384482966b9bc577c65248bda910e82f3 100644 --- a/lib/sse_t1/crc32_iuup_sse.asm +++ b/lib/sse_t1/crc32_iuup_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/crc32_lte_sse.asm b/lib/sse_t1/crc32_lte_sse.asm index 64d7ae7eb8dd56d1c85fadee9457db7e88b8cf81..f490d4dbe6fa1a1ba1dd7c506bc9c2cf96958ece 100644 --- a/lib/sse_t1/crc32_lte_sse.asm +++ b/lib/sse_t1/crc32_lte_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/crc32_refl_by8_sse.asm b/lib/sse_t1/crc32_refl_by8_sse.asm index b5ab5860647a85526e3419be736d967f89ee2257..39d2006942b22abe872f2926f4c85b5edc471840 100644 --- a/lib/sse_t1/crc32_refl_by8_sse.asm +++ b/lib/sse_t1/crc32_refl_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -35,11 +35,11 @@ ;; "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" ;; URL: http://download.intel.com/design/intarch/papers/323102.pdf -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/reg_sizes.inc" %include "include/crc32_refl.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifndef CRC32_REFL_FN %define CRC32_REFL_FN crc32_refl_by8_sse diff --git a/lib/sse_t1/crc32_sctp_sse.asm b/lib/sse_t1/crc32_sctp_sse.asm index 3d17ac6a5734138372b19f86c4975cc70cf38c5b..a2fa5b43d2b0a17151ff7eede9beede0e792370c 100644 --- a/lib/sse_t1/crc32_sctp_sse.asm +++ b/lib/sse_t1/crc32_sctp_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/crc32_wimax_sse.asm b/lib/sse_t1/crc32_wimax_sse.asm index e89b0d06e355a744e1a039d011ebcda617f94cd0..be8e0ba1d095d3e07ed0cbd138ac8e7ea95da25c 100644 --- a/lib/sse_t1/crc32_wimax_sse.asm +++ b/lib/sse_t1/crc32_wimax_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/ethernet_fcs_sse.asm b/lib/sse_t1/ethernet_fcs_sse.asm index 74c10ecf5ec4ff7307c79e572ffb3ce792abad86..c8271fa916ed0c2d2f6b355f782b1b312366316a 100644 --- a/lib/sse_t1/ethernet_fcs_sse.asm +++ b/lib/sse_t1/ethernet_fcs_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/crc32_refl_const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/gcm128_api_by8_sse.asm b/lib/sse_t1/gcm128_api_by8_sse.asm index 96e7a3409d5b9cdc3878c94e5d4ec6c517e588f8..b2ec5cbe0860b629c41271748bc129e886ed4522 100644 --- a/lib/sse_t1/gcm128_api_by8_sse.asm +++ b/lib/sse_t1/gcm128_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "sse_t1/gcm_api_sse.inc" +%include "include/gcm_api_sse.inc" diff --git a/lib/sse_t1/gcm128_gmac_api_by8_sse.asm b/lib/sse_t1/gcm128_gmac_api_by8_sse.asm index e222cf4e297b756ec0c304f10eb101706d2dc246..8674746dc835865eebae22a341d99f5814c745b5 100644 --- a/lib/sse_t1/gcm128_gmac_api_by8_sse.asm +++ b/lib/sse_t1/gcm128_gmac_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "sse_t1/gcm_gmac_api_sse.inc" +%include "include/gcm_gmac_api_sse.inc" diff --git a/lib/sse_t1/gcm128_sgl_api_by8_sse.asm b/lib/sse_t1/gcm128_sgl_api_by8_sse.asm index f6df66f7557b35bea64f33ff88f7a180f254aa9c..3ddbbc6590bdccd80ec9333910e8f86bb83755f6 100644 --- a/lib/sse_t1/gcm128_sgl_api_by8_sse.asm +++ b/lib/sse_t1/gcm128_sgl_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "sse_t1/gcm_sgl_api_sse.inc" +%include "include/gcm_sgl_api_sse.inc" diff --git a/lib/sse_t1/gcm192_api_by8_sse.asm b/lib/sse_t1/gcm192_api_by8_sse.asm index 13cf48d4332040c4dc36edaed449a9b269b80f69..cac47d103796dfcd6ff96b0ed5fc790d65185ef4 100644 --- a/lib/sse_t1/gcm192_api_by8_sse.asm +++ b/lib/sse_t1/gcm192_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "sse_t1/gcm_api_sse.inc" +%include "include/gcm_api_sse.inc" diff --git a/lib/sse_t1/gcm192_gmac_api_by8_sse.asm b/lib/sse_t1/gcm192_gmac_api_by8_sse.asm index 5c9967e0bbda3f81ed63f9d47e45998b4742f7e8..587d2b0b46e8eca6fa86f9c04950f885e77ee24a 100644 --- a/lib/sse_t1/gcm192_gmac_api_by8_sse.asm +++ b/lib/sse_t1/gcm192_gmac_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "sse_t1/gcm_gmac_api_sse.inc" +%include "include/gcm_gmac_api_sse.inc" diff --git a/lib/sse_t1/gcm192_sgl_api_by8_sse.asm b/lib/sse_t1/gcm192_sgl_api_by8_sse.asm index 6b6a83572f0bfe07d574e18fcfbe99b719ff3852..b3b4e48a53c9d81e2043daa4d87a9fab3e306630 100644 --- a/lib/sse_t1/gcm192_sgl_api_by8_sse.asm +++ b/lib/sse_t1/gcm192_sgl_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "sse_t1/gcm_sgl_api_sse.inc" +%include "include/gcm_sgl_api_sse.inc" diff --git a/lib/sse_t1/gcm256_api_by8_sse.asm b/lib/sse_t1/gcm256_api_by8_sse.asm index fdffe34b5e62e75dc66bd1aa23c44ee7458b7649..4bff1931aae95b9b9bffd1a7e4ee460f54ef7106 100644 --- a/lib/sse_t1/gcm256_api_by8_sse.asm +++ b/lib/sse_t1/gcm256_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "sse_t1/gcm_api_sse.inc" +%include "include/gcm_api_sse.inc" diff --git a/lib/sse_t1/gcm256_gmac_api_by8_sse.asm b/lib/sse_t1/gcm256_gmac_api_by8_sse.asm index 5f8dc21e8bc8045ee71fddbfbe425a62ef849377..c8678ee65c6fc64379801d8a9914513c7a2e0ad6 100644 --- a/lib/sse_t1/gcm256_gmac_api_by8_sse.asm +++ b/lib/sse_t1/gcm256_gmac_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "sse_t1/gcm_gmac_api_sse.inc" +%include "include/gcm_gmac_api_sse.inc" diff --git a/lib/sse_t1/gcm256_sgl_api_by8_sse.asm b/lib/sse_t1/gcm256_sgl_api_by8_sse.asm index 304b3cb1f70a6fb3e197edc0c2df1632b20c367d..9bf49c62ef9324e52823038e586f9e1b7d4c23f8 100644 --- a/lib/sse_t1/gcm256_sgl_api_by8_sse.asm +++ b/lib/sse_t1/gcm256_sgl_api_by8_sse.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022 Intel Corporation All rights reserved. +; Copyright(c) 2021-2023 Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "sse_t1/gcm_sgl_api_sse.inc" +%include "include/gcm_sgl_api_sse.inc" diff --git a/lib/sse_t1/kasumi_sse.c b/lib/sse_t1/kasumi_sse.c index 7bfecce4bc4334e173ce2396cad72718bb7c4c74..3a63917ba8cb1d04237b00348996b8e1c502e08d 100644 --- a/lib/sse_t1/kasumi_sse.c +++ b/lib/sse_t1/kasumi_sse.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm index 35362ca7012979d82bceead67e300073f499d469..c095f759acfc4e8ab09f19665eb02708d10e0915 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" %ifndef NUM_LANES %define NUM_LANES 4 diff --git a/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm index 0c67da092541157ff8889b224d734b315749c409..b95ec3cdda3a5458325997072c5b550ee5e45c67 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" -%include "include/reg_sizes.asm" +%include "include/reg_sizes.inc" %include "include/const.inc" %ifndef NUM_LANES %define NUM_LANES 4 diff --git a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm index 54ec68b45c8bb878904f36005e00cb2799f4ebed..3ede4b67c1a92336be1f89ba7a548403c0d271a6 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %define NUM_LANES 4 diff --git a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm index 396fb20222fbb6293d4b02d0fc7d5aca438ce171..815962d48dc1898229fc6d1323fea26a5a3e6902 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" %define NUM_LANES 4 diff --git a/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm index 7e12dedd6c903db6691f1fc6c11fd5c37b18ec85..638f50b877fd283ebc6210b403df3cf4db40edbf 100644 --- a/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/memcpy.asm" +%include "include/memcpy.inc" %ifndef NUM_LANES %define NUM_LANES 4 diff --git a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm index be832f5b29eb254ff980a00d60fa7b8802a1b9c9..dfcc08dbc38529ddfec6ab8760b887ae156be8a4 100644 --- a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,15 +25,15 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" %ifndef NUM_LANES %define NUM_LANES 4 diff --git a/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm index 8d765e9980f3eea0ca492b39210151efaf63aeb3..317aae1716b2ec757ce97afa12a0916803651627 100644 --- a/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %ifndef AES_XCBC_X4 %define AES_XCBC_X4 aes_xcbc_mac_128_x4 diff --git a/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm index 73afe3e525a7675aff6050dbe708a11b9f4fb61b..541acaeb924e168fb0c7102052b1754e7b6496b6 100644 --- a/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" %include "include/const.inc" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %ifndef AES_XCBC_X4 %define AES_XCBC_X4 aes_xcbc_mac_128_x4 %define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse diff --git a/lib/sse_t1/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm index b0e89607fe4e77825f3af36d6a8178edb5d28852..7b82575496a03863bd0f74172375b88a9f31c787 100644 --- a/lib/sse_t1/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm index 4ce3d952d4a50daeeecea6737749d18265aaa6a6..e6674529aa7a6104f2b17aa2c4f78664aabf6d97 100644 --- a/lib/sse_t1/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm index 5c82dd21afc338b08483107d7ccc5f0b972947cd..f5ebb1c8d9467b5d81a4857b9d6f586a865bd261 100644 --- a/lib/sse_t1/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm index 3736b445053889c4bdf78514d0c0ad0041730ac9..3489895b9305254409daad2f89aca148ab9c6ffa 100644 --- a/lib/sse_t1/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm index 6dce9dfa954312f2fa2cafc58ecf04664086b3ac..efe85e77a633dfeb83a2a385af083b2666e4254d 100644 --- a/lib/sse_t1/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm index c7c33d976bd7ebcf82db44dc71a1110bbfddc548..e05b9d6e4eb274ba75cdf8d4b53b98c7f60b9590 100644 --- a/lib/sse_t1/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_hmac_md5_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_md5_flush_sse.asm index 5a95d448b7088729b526433af203c14fe5592cc1..c0aa2bab8ea4a2f1d875fb4dcb19616512b84834 100644 --- a/lib/sse_t1/mb_mgr_hmac_md5_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_md5_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern md5_x4x2_sse diff --git a/lib/sse_t1/mb_mgr_hmac_md5_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_md5_submit_sse.asm index 0729251fea11cf295931d5c0af80affaf3ab03b5..bff94e799bb7dc72cd1ff9dd3d67cad87d4c3349 100644 --- a/lib/sse_t1/mb_mgr_hmac_md5_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_md5_submit_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/memcpy.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/memcpy.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" extern md5_x4x2_sse diff --git a/lib/sse_t1/mb_mgr_hmac_sha1_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha1_flush_sse.asm index 6d5789358c090109fd72b929eb0b6ba77334b9fa..57cde6d6f81164e4b9eda2861e18d262215d2885 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha1_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha1_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,13 +25,13 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha1_mult_sse diff --git a/lib/sse_t1/mb_mgr_hmac_sha1_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha1_submit_sse.asm index b8a487bea8a766b661102a128cadb4da1ec0c308..b76b3ff295e5eb2c3aae35ef538b30f9e0996a74 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha1_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha1_submit_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,15 +25,15 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha1_mult_sse diff --git a/lib/sse_t1/mb_mgr_hmac_sha224_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha224_flush_sse.asm index bda204599baef127e0e6d4e8477e1c63f3f72081..e8d05d3124d327d2e6a6d8ed6eb89d562e0f6789 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha224_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha224_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_hmac_sha224_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha224_submit_sse.asm index bddf843ea43074ea99e0e34f1afe72def80efd60..6fc547f15f2dd5e9408b3e72e7f10d59b08aa697 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha224_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha224_submit_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_hmac_sha256_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha256_flush_sse.asm index 63d94c15b589e0b6cb3238a5e796fe41dabd315f..877530844aa2f626af3458e82713b28c0a6c5aae 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha256_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha256_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern sha_256_mult_sse diff --git a/lib/sse_t1/mb_mgr_hmac_sha256_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha256_submit_sse.asm index 7a8e6aa4e24565ed690f16ec41ec8efc1695983e..d3d356152c89c4e48a6e2e9f08c25f8e1949fd20 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha256_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha256_submit_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" extern sha_256_mult_sse diff --git a/lib/sse_t1/mb_mgr_hmac_sha384_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha384_flush_sse.asm index 8e95926a933a56a5b8cb8d21a8a716bb07399c88..2ffc76b005d802f0c9748db698aeab83df1d8d81 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha384_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha384_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_hmac_sha384_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha384_submit_sse.asm index 3734957189bae23edaf69307a43f6d0395e3b9be..dd5d4ecdca72722a5368cc5940e688750b1fa021 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha384_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha384_submit_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_hmac_sha512_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha512_flush_sse.asm index 0b6d1be0793211b9c26f4abb64c1006b8b298113..903a028244fadfe7d5530dd08fd2f50ac42e9e0c 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha512_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha512_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" extern sha512_x2_sse diff --git a/lib/sse_t1/mb_mgr_hmac_sha512_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha512_submit_sse.asm index 78a6778b7d44f4ea1abf1869625711f2ef3b5e72..208dde6f8a98b27f6b3e1ea2c43e44ae3d6f74a7 100644 --- a/lib/sse_t1/mb_mgr_hmac_sha512_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha512_submit_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" %include "include/const.inc" extern sha512_x2_sse diff --git a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm index d06c403ff3765a96fd5f3c5192178aac4419a8a4..f94ae93e6b30e73605daa14da5120281a13bf691 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "sse_t1/snow3g_uea2_by4_sse.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/snow3g_uea2_by4_sse.inc" %define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_sse %define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_sse diff --git a/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm index 4490a7485d9f4fcbc3628b269e9d290816334024..aff4de7979f09b070fe102eb42f6bfa368f9ddd9 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/clear_regs.asm" -%include "sse_t1/snow3g_uea2_by4_sse.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/clear_regs.inc" +%include "include/snow3g_uea2_by4_sse.inc" %define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse %define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index fe09e2f423ffbb048bc33719af7689a5df885a4c..d53c4a5db1674513cbce2fd7b2defdaa642e3892 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/mb_mgr_sse_t1.c b/lib/sse_t1/mb_mgr_sse_t1.c index afbaf667160c6d105e02c44fb80eefb60a8eed8b..bbab72d2d5334c61d43825ce97ea397337089125 100644 --- a/lib/sse_t1/mb_mgr_sse_t1.c +++ b/lib/sse_t1/mb_mgr_sse_t1.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -70,6 +70,7 @@ #define SUBMIT_HASH_BURST submit_hash_burst_sse_t1 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_t1 +#define SET_SUITE_ID_FN set_suite_id_sse_t1 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE @@ -366,6 +367,7 @@ init_mb_mgr_sse_t1_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_sse; state->keyexp_192 = aes_keyexp_192_sse; @@ -390,6 +392,7 @@ init_mb_mgr_sse_t1_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_sse; state->aes128_cfb_one = aes_cfb_128_one_sse; + state->aes256_cfb_one = aes_cfb_256_one_sse; state->eea3_1_buffer = zuc_eea3_1_buffer_sse; state->eea3_4_buffer = zuc_eea3_4_buffer_sse; @@ -485,6 +488,9 @@ init_mb_mgr_sse_t1_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_sse; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_sse; } #include "mb_mgr_code.h" diff --git a/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm b/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm index efc6f9a713aaec8c891a05853b0d68fa606096e5..0aa664b19b78777d7f66448b82948ca2763742d3 100644 --- a/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" %include "include/const.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifndef SUBMIT_JOB_ZUC128_EEA3 %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_no_gfni_sse diff --git a/lib/sse_t1/md5_x4x2_sse.asm b/lib/sse_t1/md5_x4x2_sse.asm index b0d568d6f9b3c3dc21deed636e864023dddba845..64f7af8e09d87f813e423022414730e1753e44c7 100644 --- a/lib/sse_t1/md5_x4x2_sse.asm +++ b/lib/sse_t1/md5_x4x2_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -36,9 +36,9 @@ ;; ;; clobbers xmm0-15 -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" mksection .rodata align=64 default rel diff --git a/lib/sse_t1/memcpy_sse.asm b/lib/sse_t1/memcpy_sse.asm index b2433175f0b637dda31b38aca0396a2cbcab13f3..769790a099047f68242913d9844633f582db92eb 100644 --- a/lib/sse_t1/memcpy_sse.asm +++ b/lib/sse_t1/memcpy_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,8 +25,8 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/memcpy.inc" %ifdef LINUX %define arg1 rdi diff --git a/lib/sse_t1/pon_by8_sse.asm b/lib/sse_t1/pon_by8_sse.asm index f0a4f0b52266f421d3e79446d14e18c23ab04c66..948a95f767f796f650efa4d63c5520f55c329cb3 100644 --- a/lib/sse_t1/pon_by8_sse.asm +++ b/lib/sse_t1/pon_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/imb_job.asm" -%include "include/os.asm" -%include "include/memcpy.asm" -%include "include/clear_regs.asm" +%include "include/imb_job.inc" +%include "include/os.inc" +%include "include/memcpy.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" diff --git a/lib/sse_t1/sha1_one_block_sse.asm b/lib/sse_t1/sha1_one_block_sse.asm index 48564a67ee8f05dc3d4be14296f9ab411f4fd4ce..2893483d73bb7ceba7c1edfcf7abbd7d03bcb66b 100644 --- a/lib/sse_t1/sha1_one_block_sse.asm +++ b/lib/sse_t1/sha1_one_block_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,8 +27,8 @@ ; SHA1 code, hybrid, rolled, interleaved ; Uses SSE instructions -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/sse_t1/sha1_x4_sse.asm b/lib/sse_t1/sha1_x4_sse.asm index 7994bc1be0eb6e82ee34c83c6b0b0686ba0621e0..2c755749c11e59936ee4b6982bd9c3c21cbd9799 100644 --- a/lib/sse_t1/sha1_x4_sse.asm +++ b/lib/sse_t1/sha1_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,12 +25,12 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/sse_t1/sha224_one_block_sse.asm b/lib/sse_t1/sha224_one_block_sse.asm index cebbe79f5bab45ec3e21eb891e7e500aa7a76f25..b83b2bc64bf328a4fbdd8313df81e8943fc7bb2a 100644 --- a/lib/sse_t1/sha224_one_block_sse.asm +++ b/lib/sse_t1/sha224_one_block_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/sha256_mult_sse.asm b/lib/sse_t1/sha256_mult_sse.asm index e77fcd27bf969b560c1265d23191a48cbbd0541f..7179bcd075cce3f9f311c082f8ffc3f9543da467 100644 --- a/lib/sse_t1/sha256_mult_sse.asm +++ b/lib/sse_t1/sha256_mult_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -38,12 +38,12 @@ ;; ;; clobbers xmm0-15 -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" mksection .rodata default rel diff --git a/lib/sse_t1/sha256_one_block_sse.asm b/lib/sse_t1/sha256_one_block_sse.asm index 74872e41fe8e609989d7b22cea5a612456f90e12..2c42280aef15e45ce699500e2d00774385f1c837 100644 --- a/lib/sse_t1/sha256_one_block_sse.asm +++ b/lib/sse_t1/sha256_one_block_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,8 +27,8 @@ ; This code schedules 1 blocks at a time, with 4 lanes per block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" mksection .rodata default rel diff --git a/lib/sse_t1/sha384_one_block_sse.asm b/lib/sse_t1/sha384_one_block_sse.asm index 9db43fd9268202649de28b59d422796574dc303d..6f3bef520920266d6ec2eec16b1ecc12e40277b8 100644 --- a/lib/sse_t1/sha384_one_block_sse.asm +++ b/lib/sse_t1/sha384_one_block_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/sha512_one_block_sse.asm b/lib/sse_t1/sha512_one_block_sse.asm index bc5a26fb632443027ee99583aa0aee5dcc8710e9..b63766c4feb61f234ae4f88ba3200c6e2cf0e870 100644 --- a/lib/sse_t1/sha512_one_block_sse.asm +++ b/lib/sse_t1/sha512_one_block_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,8 +27,8 @@ ; This code schedules 1 blocks at a time, with 4 lanes per block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %define MOVDQ movdqu ;; assume buffers not aligned diff --git a/lib/sse_t1/sha512_x2_sse.asm b/lib/sse_t1/sha512_x2_sse.asm index 999b75a315f2234e0e26b6558b61deb4ed196ac1..c209e2dd3e42b51871064bb08ee9e2304a0b4248 100644 --- a/lib/sse_t1/sha512_x2_sse.asm +++ b/lib/sse_t1/sha512_x2_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -39,12 +39,12 @@ ;; ;; clobbers xmm0-15 -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" mksection .rodata default rel diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index e528e6ff033c1b03e9ba219b7ed4df27edcf4669..c5d91822c391530bcf95cad451e4fb5510fc454f 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/sha_sse.c b/lib/sse_t1/sha_sse.c index 40ba89c72cdab43665aa256865e5de1356eaec3f..e61f6e58a74c75a865cb8dc43333699d27a7edd3 100644 --- a/lib/sse_t1/sha_sse.c +++ b/lib/sse_t1/sha_sse.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -26,13 +26,14 @@ *******************************************************************************/ #include "include/sha_generic.h" +#include "include/arch_sse_type1.h" /* ========================================================================== */ /* One block SHA1 computation for IPAD / OPAD usage only */ void sha1_one_block_sse(const void *data, void *digest) { - sha_generic_1block(data, digest, 0 /* SSE */, 1 /* SHA1 */); + sha_generic_1block(data, digest, ARCH_SSE, 1 /* SHA1 */); } /* ========================================================================== */ @@ -42,7 +43,7 @@ void sha1_one_block_sse(const void *data, void *digest) void sha1_sse(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 0 /* SSE */, 1, IMB_SHA1_BLOCK_SIZE, + sha_generic(data, length, digest, ARCH_SSE, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE); } @@ -51,7 +52,7 @@ void sha1_sse(const void *data, const uint64_t length, void *digest) void sha224_one_block_sse(const void *data, void *digest) { - sha_generic_1block(data, digest, 0 /* SSE */, 224 /* SHA224 */); + sha_generic_1block(data, digest, ARCH_SSE, 224 /* SHA224 */); } /* ========================================================================== */ @@ -60,7 +61,7 @@ void sha224_one_block_sse(const void *data, void *digest) */ void sha224_sse(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 0 /* SSE */, 224, + sha_generic(data, length, digest, ARCH_SSE, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE); } @@ -69,7 +70,7 @@ void sha224_sse(const void *data, const uint64_t length, void *digest) void sha256_one_block_sse(const void *data, void *digest) { - sha_generic_1block(data, digest, 0 /* SSE */, 256 /* SHA256 */); + sha_generic_1block(data, digest, ARCH_SSE, 256 /* SHA256 */); } /* ========================================================================== */ @@ -78,7 +79,7 @@ void sha256_one_block_sse(const void *data, void *digest) */ void sha256_sse(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 0 /* SSE */, 256, + sha_generic(data, length, digest, ARCH_SSE, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE); } @@ -87,7 +88,7 @@ void sha256_sse(const void *data, const uint64_t length, void *digest) void sha384_one_block_sse(const void *data, void *digest) { - sha_generic_1block(data, digest, 0 /* SSE */, 384 /* SHA384 */); + sha_generic_1block(data, digest, ARCH_SSE, 384 /* SHA384 */); } /* ========================================================================== */ @@ -96,7 +97,7 @@ void sha384_one_block_sse(const void *data, void *digest) */ void sha384_sse(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 0 /* SSE */, 384, + sha_generic(data, length, digest, ARCH_SSE, 384, IMB_SHA_384_BLOCK_SIZE, SHA384_PAD_SIZE); } @@ -105,7 +106,7 @@ void sha384_sse(const void *data, const uint64_t length, void *digest) void sha512_one_block_sse(const void *data, void *digest) { - sha_generic_1block(data, digest, 0 /* SSE */, 512 /* SHA512 */); + sha_generic_1block(data, digest, ARCH_SSE, 512 /* SHA512 */); } /* ========================================================================== */ @@ -114,6 +115,6 @@ void sha512_one_block_sse(const void *data, void *digest) */ void sha512_sse(const void *data, const uint64_t length, void *digest) { - sha_generic(data, length, digest, 0 /* SSE */, 512, + sha_generic(data, length, digest, ARCH_SSE, 512, IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE); } diff --git a/lib/sse_t1/snow3g_sse.c b/lib/sse_t1/snow3g_sse.c index 664007b07e79ec5e127150e80efa00a3c8db183a..8c54d44533ece729a5116af53034767dfa953c59 100644 --- a/lib/sse_t1/snow3g_sse.c +++ b/lib/sse_t1/snow3g_sse.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/snow3g_uia2_by4_sse.asm b/lib/sse_t1/snow3g_uia2_by4_sse.asm index 2e0be5706a218cfc997b41db27873c14463f1307..ee5369b65b3f7f91f17651756dc5b497bcefd1e0 100644 --- a/lib/sse_t1/snow3g_uia2_by4_sse.asm +++ b/lib/sse_t1/snow3g_uia2_by4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/cet.inc" -%include "include/memcpy.asm" +%include "include/memcpy.inc" %include "include/const.inc" %define APPEND(a,b) a %+ b %define APPEND3(a,b,c) a %+ b %+ c diff --git a/lib/sse_t1/snow_v_sse.asm b/lib/sse_t1/snow_v_sse.asm index c330c8bb00063b63881a2206da8f210bf9e7c8c3..d4118430215c74de804a0120cecb40061520f9fc 100644 --- a/lib/sse_t1/snow_v_sse.asm +++ b/lib/sse_t1/snow_v_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2021-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -30,11 +30,11 @@ ;; abstract : 'A new SNOW stream cipher called SNOW-V' ;; https://eprint.iacr.org/2018/1143.pdf -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" -%include "include/imb_job.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" %ifndef SNOW_V @@ -75,15 +75,21 @@ dq 0x20646b4578656c41 dq 0x6d6f6854676E694a %ifdef LINUX - %define arg1 rdi - %define offset rcx + %define arg1 rdi + %define offset rcx %else - %define arg1 rcx - %define offset r8 + %define arg1 rcx + %define offset r8 %endif %define job arg1 +;; stack frame for saving registers (windows only) +struc STACK +_xmm_save: resq 10 * 2 ; space for 10 xmm registers +_rsp_save: resq 1 ; space for rsp pointer +endstruc + mksection .text ;; Registers usage @@ -93,20 +99,20 @@ mksection .text ;; xmm8, xmm9, xmm10, xmm11 : LFSR_A, LFSR_B ;; xmm13, xmm14, xmm15 : constants gA, gB, inv_gA -%define gA xmm13 -%define gB xmm14 -%define inv_gA xmm15 +%define gA xmm13 +%define gB xmm14 +%define inv_gA xmm15 -%define FSM_R1 xmm5 -%define FSM_R2 xmm6 -%define FSM_R3 xmm7 +%define FSM_R1 xmm5 +%define FSM_R2 xmm6 +%define FSM_R3 xmm7 -%define LFSR_A_LDQ xmm8 ;; LSFR A: (a7, ..., a0) -%define LFSR_A_HDQ xmm9 ;; LSFR A: (a15, ..., a8) -%define LFSR_B_LDQ xmm10 ;; LSFR B: (b7, ..., b0) -%define LFSR_B_HDQ xmm11 ;; LSFR B: (b15, ..., b8) +%define LFSR_A_LDQ xmm8 ;; LSFR A: (a7, ..., a0) +%define LFSR_A_HDQ xmm9 ;; LSFR A: (a15, ..., a8) +%define LFSR_B_LDQ xmm10 ;; LSFR B: (b7, ..., b0) +%define LFSR_B_HDQ xmm11 ;; LSFR B: (b15, ..., b8) -%define temp4 xmm12 +%define temp4 xmm12 ;; ============================================================================= ;; ============================================================================= @@ -120,9 +126,9 @@ mksection .text %define %%FSM_R1 %3 ;; [in] 128 bit FSM: R1 %define %%FSM_R2 %4 ;; [in] 128 bit FSM: R2 - movdqa %%KEYSTREAM, %%LFSR_B_HDQ - paddd %%KEYSTREAM, %%FSM_R1 - pxor %%KEYSTREAM, %%FSM_R2 + movdqa %%KEYSTREAM, %%LFSR_B_HDQ + paddd %%KEYSTREAM, %%FSM_R1 + pxor %%KEYSTREAM, %%FSM_R2 %endmacro ;; SNOW_V_KEYSTREAM @@ -142,17 +148,17 @@ mksection .text ;; containing copy of LFSR_A_LDQ (a7, ..., a0) %define %%TEMP1 %5 ;; [clobbered] 128 bit register - pxor %%T2, %%FSM_R3 ;; T2 = R3 XOR LSFR_A [0:7] - paddd %%T2, %%FSM_R2 ;; T2 += R2 - pshufb %%T2, [rel sigma] ;; T2 = sigma(T2) + pxor %%T2, %%FSM_R3 ;; T2 = R3 XOR LSFR_A [0:7] + paddd %%T2, %%FSM_R2 ;; T2 += R2 + pshufb %%T2, [rel sigma] ;; T2 = sigma(T2) - movdqa %%FSM_R3, %%FSM_R2 ;; R3 = R2 - movdqa %%FSM_R2, %%FSM_R1 ;; R2 = R1 - pxor %%TEMP1, %%TEMP1 ;; TEMP1 = 0 + movdqa %%FSM_R3, %%FSM_R2 ;; R3 = R2 + movdqa %%FSM_R2, %%FSM_R1 ;; R2 = R1 + pxor %%TEMP1, %%TEMP1 ;; TEMP1 = 0 - movdqa %%FSM_R1, %%T2 ;; R1 = sigma(T2) - aesenc %%FSM_R3, %%TEMP1 ;; R3 = AESR(R2) (encryption round key C1 = 0) - aesenc %%FSM_R2, %%TEMP1 ;; R2 = AESR(R1) (encryption round key C2 = 0) + movdqa %%FSM_R1, %%T2 ;; R1 = sigma(T2) + aesenc %%FSM_R3, %%TEMP1 ;; R3 = AESR(R2) (encryption round key C1 = 0) + aesenc %%FSM_R2, %%TEMP1 ;; R2 = AESR(R1) (encryption round key C2 = 0) %endmacro ;; SNOW_V_FSM_UPDATE @@ -191,207 +197,247 @@ mksection .text ;; else: (x >> 1) ;; = (x >> 1) xor signw(inv_gA, x << 15) - ;; calculate mulx_A = (alpha*a7, ..., alpha*a0) - movdqa %%TEMP1, %%LFSR_A_LDQ - psraw %%TEMP1, 15 ;; 16-bit mask with sign bits preserved - pand %%TEMP1, %%gA - movdqa %%TEMP2, %%LFSR_A_LDQ - psllw %%TEMP2, 1 - pxor %%TEMP2, %%TEMP1 ;; TEMP2 = mulx_A - - ;; calculate invx_A = (alpha^-1*a15, ..., alpha^-1*a8) - movdqa %%TEMP1, %%LFSR_A_HDQ - psllw %%TEMP1, 15 - movdqa %%T2, %%inv_gA - psignw %%T2, %%TEMP1 ;; negate bits in inv_gA depending on LFSR_A_HDQ << 15 - movdqa %%TEMP1, %%LFSR_A_HDQ - psrlw %%TEMP1, 1 - pxor %%TEMP1, %%T2 ;; TEMP1 = invx_A - - movdqa %%T2, %%LFSR_A_HDQ ;; make copy of LFSR_A_HDQ - - ;; LFSR_A_HDQ = mulx_A XOR invx_A XOR (b7, ..., b0) XOR (a8, ..., a1) - pxor %%TEMP1, %%TEMP2 ;; TEMP1 = invx_A xor mulx_A - palignr %%LFSR_A_HDQ, %%LFSR_A_LDQ, 2 ;; T2 = (tmpa_8, ..., tmpa_1) - pxor %%LFSR_A_HDQ, %%LFSR_B_LDQ - pxor %%LFSR_A_HDQ, %%TEMP1 - - ;; calculate mulx_B - movdqa %%TEMP1, %%LFSR_B_LDQ - psraw %%TEMP1, 15 - pand %%TEMP1, %%gB - movdqa %%TEMP2, %%LFSR_B_LDQ - psllw %%TEMP2, 1 - pxor %%TEMP1, %%TEMP2 - - ;; T1 = mulx_B XOR (a7, ..., a0) XOR (b10, ..., b3) - pxor %%TEMP1, %%LFSR_A_LDQ - palignr %%T1, %%LFSR_B_LDQ, 6 ;; (b10, ..., b3) - pxor %%T1, %%TEMP1 - - ;; calculate invx_B - movdqa %%TEMP1, %%LFSR_B_HDQ - psllw %%TEMP1, 15 - movdqa %%TEMP2, [rel beta_inv] - psignw %%TEMP2, %%TEMP1 - movdqa %%TEMP1, %%LFSR_B_HDQ - psrlw %%TEMP1, 1 - pxor %%TEMP1, %%TEMP2 - - ;; LFSR_B_HDQ = mulx_B XOR invx_B XOR (a7, ..., a0) XOR (b10, ..., b3) - pxor %%T1, %%TEMP1 - - movdqa %%LFSR_B_LDQ, %%LFSR_B_HDQ ;; LFSR_B_LDQ = LFSR_B_HDQ - movdqa %%LFSR_A_LDQ, %%T2 ;; LFSR_A_LDQ = LFSR_A_HDQ - movdqa %%LFSR_B_HDQ, %%T1 + ;; calculate mulx_A = (alpha*a7, ..., alpha*a0) + movdqa %%TEMP1, %%LFSR_A_LDQ + psraw %%TEMP1, 15 ;; 16-bit mask with sign bits preserved + pand %%TEMP1, %%gA + movdqa %%TEMP2, %%LFSR_A_LDQ + psllw %%TEMP2, 1 + pxor %%TEMP2, %%TEMP1 ;; TEMP2 = mulx_A + + ;; calculate invx_A = (alpha^-1*a15, ..., alpha^-1*a8) + movdqa %%TEMP1, %%LFSR_A_HDQ + psllw %%TEMP1, 15 + movdqa %%T2, %%inv_gA + psignw %%T2, %%TEMP1 ;; negate bits in inv_gA depending on LFSR_A_HDQ << 15 + movdqa %%TEMP1, %%LFSR_A_HDQ + psrlw %%TEMP1, 1 + pxor %%TEMP1, %%T2 ;; TEMP1 = invx_A + + movdqa %%T2, %%LFSR_A_HDQ ;; make copy of LFSR_A_HDQ + + ;; LFSR_A_HDQ = mulx_A XOR invx_A XOR (b7, ..., b0) XOR (a8, ..., a1) + pxor %%TEMP1, %%TEMP2 ;; TEMP1 = invx_A xor mulx_A + palignr %%LFSR_A_HDQ, %%LFSR_A_LDQ, 2 ;; T2 = (tmpa_8, ..., tmpa_1) + pxor %%LFSR_A_HDQ, %%LFSR_B_LDQ + pxor %%LFSR_A_HDQ, %%TEMP1 + + ;; calculate mulx_B + movdqa %%TEMP1, %%LFSR_B_LDQ + psraw %%TEMP1, 15 + pand %%TEMP1, %%gB + movdqa %%TEMP2, %%LFSR_B_LDQ + psllw %%TEMP2, 1 + pxor %%TEMP1, %%TEMP2 + + ;; T1 = mulx_B XOR (a7, ..., a0) XOR (b10, ..., b3) + pxor %%TEMP1, %%LFSR_A_LDQ + palignr %%T1, %%LFSR_B_LDQ, 6 ;; (b10, ..., b3) + pxor %%T1, %%TEMP1 + + ;; calculate invx_B + movdqa %%TEMP1, %%LFSR_B_HDQ + psllw %%TEMP1, 15 + movdqa %%TEMP2, [rel beta_inv] + psignw %%TEMP2, %%TEMP1 + movdqa %%TEMP1, %%LFSR_B_HDQ + psrlw %%TEMP1, 1 + pxor %%TEMP1, %%TEMP2 + + ;; LFSR_B_HDQ = mulx_B XOR invx_B XOR (a7, ..., a0) XOR (b10, ..., b3) + pxor %%T1, %%TEMP1 + + movdqa %%LFSR_B_LDQ, %%LFSR_B_HDQ ;; LFSR_B_LDQ = LFSR_B_HDQ + movdqa %%LFSR_A_LDQ, %%T2 ;; LFSR_A_LDQ = LFSR_A_HDQ + movdqa %%LFSR_B_HDQ, %%T1 %endmacro ;; SNOW_V_LFSR_UPDATE ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro FUNC_START 0 +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + mov rax, rsp + sub rsp, STACK_size + and rsp, -16 + mov [rsp + _rsp_save], rax + movdqa [rsp + _xmm_save + 0*16], xmm6 + movdqa [rsp + _xmm_save + 1*16], xmm7 + movdqa [rsp + _xmm_save + 2*16], xmm8 + movdqa [rsp + _xmm_save + 3*16], xmm9 + movdqa [rsp + _xmm_save + 4*16], xmm10 + movdqa [rsp + _xmm_save + 5*16], xmm11 + movdqa [rsp + _xmm_save + 6*16], xmm12 + movdqa [rsp + _xmm_save + 7*16], xmm13 + movdqa [rsp + _xmm_save + 8*16], xmm14 + movdqa [rsp + _xmm_save + 9*16], xmm15 +%endif +%endmacro + +%macro FUNC_END 0 +%ifidn __OUTPUT_FORMAT__, win64 + movdqa xmm6, [rsp + _xmm_save + 0*16] + movdqa xmm7, [rsp + _xmm_save + 1*16] + movdqa xmm8, [rsp + _xmm_save + 2*16] + movdqa xmm9, [rsp + _xmm_save + 3*16] + movdqa xmm10, [rsp + _xmm_save + 4*16] + movdqa xmm11, [rsp + _xmm_save + 5*16] + movdqa xmm12, [rsp + _xmm_save + 6*16] + movdqa xmm13, [rsp + _xmm_save + 7*16] + movdqa xmm14, [rsp + _xmm_save + 8*16] + movdqa xmm15, [rsp + _xmm_save + 9*16] + mov rsp, [rsp + _rsp_save] +%endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + MKGLOBAL(SNOW_V_AEAD_INIT,function,) SNOW_V_AEAD_INIT: - endbranch64 - ;; use offset to indicate AEAD mode - mov DWORD(offset), 1 - movdqa LFSR_B_LDQ, [rel aead_lsfr_b_lo] - jmp snow_v_common_init + endbranch64 + FUNC_START + ;; use offset to indicate AEAD mode + mov DWORD(offset), 1 + movdqa LFSR_B_LDQ, [rel aead_lsfr_b_lo] + jmp snow_v_common_init ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(SNOW_V,function,) SNOW_V: - endbranch64 - ;; use offset to indicate AEAD mode - xor DWORD(offset), DWORD(offset) - pxor LFSR_B_LDQ, LFSR_B_LDQ + endbranch64 + FUNC_START + ;; use offset to indicate AEAD mode + xor DWORD(offset), DWORD(offset) + pxor LFSR_B_LDQ, LFSR_B_LDQ snow_v_common_init: - ;; Init LSFR - mov rax, [job + _enc_keys] - movdqu LFSR_A_HDQ, [rax] - movdqu LFSR_B_HDQ, [rax + 16] - mov rax, [job + _iv] - movdqu LFSR_A_LDQ, [rax] + ;; Init LSFR + mov rax, [job + _enc_keys] + movdqu LFSR_A_HDQ, [rax] + movdqu LFSR_B_HDQ, [rax + 16] + mov rax, [job + _iv] + movdqu LFSR_A_LDQ, [rax] - ;; Init FSM: R1 = R2 = R3 = 0 - pxor FSM_R1, FSM_R1 - pxor FSM_R2, FSM_R2 - pxor FSM_R3, FSM_R3 + ;; Init FSM: R1 = R2 = R3 = 0 + pxor FSM_R1, FSM_R1 + pxor FSM_R2, FSM_R2 + pxor FSM_R3, FSM_R3 - movdqa gA, [rel alpha] - movdqa gB, [rel beta] - movdqa inv_gA, [rel alpha_inv] + movdqa gA, [rel alpha] + movdqa gB, [rel beta] + movdqa inv_gA, [rel alpha_inv] - movdqa xmm0, LFSR_B_HDQ ;; init T1 for LSFR update - movdqa xmm1, LFSR_A_LDQ ;; init T2 for FSM update + movdqa xmm0, LFSR_B_HDQ ;; init T1 for LSFR update + movdqa xmm1, LFSR_A_LDQ ;; init T2 for FSM update - mov eax, 15 + mov eax, 15 init_fsm_lfsr_loop: - SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ, LFSR_B_LDQ, LFSR_B_HDQ, \ + SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ, LFSR_B_LDQ, LFSR_B_HDQ, \ xmm0, xmm1, xmm2, xmm3, gA, gB, inv_gA - pxor LFSR_A_HDQ, xmm4 - dec eax - jnz init_fsm_lfsr_loop + pxor LFSR_A_HDQ, xmm4 + dec eax + jnz init_fsm_lfsr_loop - mov rax, [job + _enc_keys] - movdqu temp4, [rax] - pxor FSM_R1, temp4 + mov rax, [job + _enc_keys] + movdqu temp4, [rax] + pxor FSM_R1, temp4 - SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ + SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ xmm0, xmm1, xmm2, xmm3, gA, gB, inv_gA - pxor LFSR_A_HDQ, xmm4 - movdqu temp4, [rax + 16] - pxor FSM_R1, temp4 + pxor LFSR_A_HDQ, xmm4 + movdqu temp4, [rax + 16] + pxor FSM_R1, temp4 - ;; At this point FSM and LSFR are initialized + ;; At this point FSM and LSFR are initialized - or DWORD(offset), DWORD(offset) - jz no_aead + or DWORD(offset), DWORD(offset) + jz no_aead - ;; in AEAD mode hkey = keystream_0 and endpad = keystream_1 - mov r11, [job + _snow_v_reserved] - ;; generate hkey - SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 - movdqu [r11], xmm4 + ;; in AEAD mode hkey = keystream_0 and endpad = keystream_1 + mov r11, [job + _snow_v_reserved] + ;; generate hkey + SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 + movdqu [r11], xmm4 - ;; generate endpad - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ + ;; generate endpad + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ xmm0, xmm1, xmm2, xmm3, gA, gB, inv_gA - SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 + SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ xmm0, xmm1, xmm2, xmm3, gA, gB, inv_gA - mov offset, [r11 + 24] - movdqu [r11 + 16], xmm4 - or offset, offset - ;; if last 8 bytes endpad are not 0 skip encrypt/decrypt operation - ;; option used to calculate auth tag for decrypt and not overwrite - ;; cipher by plain when the same src/dst pointer is used - jnz no_partial_block_left + mov offset, [r11 + 24] + movdqu [r11 + 16], xmm4 + or offset, offset + ;; if last 8 bytes endpad are not 0 skip encrypt/decrypt operation + ;; option used to calculate auth tag for decrypt and not overwrite + ;; cipher by plain when the same src/dst pointer is used + jnz no_partial_block_left no_aead: - ;; Process input - mov r10, [job + _src] - add r10, [job + _cipher_start_src_offset_in_bytes] - mov r11, [job + _dst] - mov rax, [job + _msg_len_to_cipher_in_bytes] - xor offset, offset - ;; deal with partial block less than 16b outside main loop - and rax, 0xfffffffffffffff0 - jz final_bytes + ;; Process input + mov r10, [job + _src] + add r10, [job + _cipher_start_src_offset_in_bytes] + mov r11, [job + _dst] + mov rax, [job + _msg_len_to_cipher_in_bytes] + xor offset, offset + ;; deal with partial block less than 16b outside main loop + and rax, 0xfffffffffffffff0 + jz final_bytes encrypt_loop: - movdqu temp4, [r10 + offset] + movdqu temp4, [r10 + offset] - SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 + SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 - SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 + SNOW_V_FSM_UPDATE FSM_R1, FSM_R2, FSM_R3, xmm1, xmm2 - SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ + SNOW_V_LFSR_UPDATE LFSR_A_LDQ, LFSR_A_HDQ,LFSR_B_LDQ, LFSR_B_HDQ, \ xmm0, xmm1, xmm2, xmm3, gA, gB, inv_gA - pxor temp4, xmm4 - movdqu [r11 + offset], temp4 - add offset, 16 - sub rax, 16 - jnz encrypt_loop + pxor temp4, xmm4 + movdqu [r11 + offset], temp4 + add offset, 16 + sub rax, 16 + jnz encrypt_loop final_bytes: - mov rax, [job + _msg_len_to_cipher_in_bytes] - and rax, 0xf - jz no_partial_block_left - - ;; load partial block into XMM register - add r10, offset - simd_load_sse_15_1 temp4, r10, rax - SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 - pxor temp4, xmm4 - add r11, offset - ;; use r10 and offset as temp [clobbered] - simd_store_sse_15 r11, temp4, rax, r10, offset + mov rax, [job + _msg_len_to_cipher_in_bytes] + and rax, 0xf + jz no_partial_block_left + + ;; load partial block into XMM register + add r10, offset + simd_load_sse_15_1 temp4, r10, rax + SNOW_V_KEYSTREAM xmm4, LFSR_B_HDQ, FSM_R1, FSM_R2 + pxor temp4, xmm4 + add r11, offset + ;; use r10 and offset as temp [clobbered] + simd_store_sse_15 r11, temp4, rax, r10, offset no_partial_block_left: - ;; Clear registers and return data + ;; Clear registers and return data %ifdef SAFE_DATA - clear_scratch_xmms_sse_asm + clear_scratch_xmms_sse_asm %endif - mov rax, job - or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER - -ret + mov rax, job + or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER + FUNC_END + ret mksection stack-noexec diff --git a/lib/sse_t1/zuc_top_sse.c b/lib/sse_t1/zuc_top_sse.c index 3217db876cb8b8e3bb1608c8550f2f76d9e93c33..e9306e56dd7e16d618975cca2168604a82eba4d7 100644 --- a/lib/sse_t1/zuc_top_sse.c +++ b/lib/sse_t1/zuc_top_sse.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t1/zuc_x4_sse.asm b/lib/sse_t1/zuc_x4_sse.asm index c14105807c7df850004b85e51e57537cbb5a7ba9..8b2d5527d180614e28e4ab45b6243ecdb143e440 100644 --- a/lib/sse_t1/zuc_x4_sse.asm +++ b/lib/sse_t1/zuc_x4_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2009-2022, Intel Corporation +;; Copyright (c) 2009-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,11 +25,11 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/zuc_sbox.inc" -%include "include/memcpy.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/memcpy.inc" +%include "include/mb_mgr_datastruct.inc" %include "include/const.inc" %ifndef ZUC_CIPHER_4 diff --git a/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm index d4a2db0d5a32c9fa050f2505cb988e4cdf530f25..3102793bda295fea5b472b80769cc80dafbda0f9 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -42,13 +42,13 @@ ;; Linux/Windows clobbers: xmm0 - xmm15 ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha1_ni_x1 diff --git a/lib/sse_t2/mb_mgr_hmac_sha1_submit_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha1_submit_ni_sse.asm index 893f209e338756ccc08951da1cf4099a97b787cc..847f4eb08b6aa7a3cee4130575cf038ed12e3d9a 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha1_submit_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha1_submit_ni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -42,14 +42,14 @@ ;; Linux/Windows clobbers: xmm0 - xmm15 ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha1_ni diff --git a/lib/sse_t2/mb_mgr_hmac_sha224_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha224_flush_ni_sse.asm index 213f5361b9662934f50e754cbd2020748af4bb4b..6aa0e7a966a0cfd9db345bd11ec320c4dee987f6 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha224_flush_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha224_flush_ni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t2/mb_mgr_hmac_sha224_submit_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha224_submit_ni_sse.asm index e95a0ccd3f60d6e0b2904554442ad3428053d516..9c7619e7c55a48f1aa9b3a5e1c92cc9f61d06e42 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha224_submit_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha224_submit_ni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm index 07b284b838d7a026749c1db57a671aafcc15987f..f5f91d0798a9bb9806f4cc41e24ce3c78d2cd2f5 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -33,13 +33,13 @@ ;; Linux/Windows clobbers: xmm0 - xmm15 ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha256_ni_x1 diff --git a/lib/sse_t2/mb_mgr_hmac_sha256_submit_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha256_submit_ni_sse.asm index 1f48c56844e11f032a48a7aa0dfcc9358dfbaab5..d44ac951d502d121ab59b869c87f330deb30e4a8 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha256_submit_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha256_submit_ni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -33,14 +33,14 @@ ;; Linux/Windows clobbers: xmm0 - xmm15 ;; -%include "include/os.asm" -%include "include/imb_job.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" +%include "include/os.inc" +%include "include/imb_job.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" +%include "include/dbgprint.inc" extern sha256_ni diff --git a/lib/sse_t2/mb_mgr_sse_t2.c b/lib/sse_t2/mb_mgr_sse_t2.c index 8ec51a180fa827dea1966181a33ff15f2b47d2f1..03e69e6b20c33eda0aa36593942a851795086d90 100644 --- a/lib/sse_t2/mb_mgr_sse_t2.c +++ b/lib/sse_t2/mb_mgr_sse_t2.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -70,6 +70,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_t2 #define SUBMIT_HASH_BURST submit_hash_burst_sse_t2 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_t2 +#define SET_SUITE_ID_FN set_suite_id_sse_t2 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE_T2 @@ -364,6 +365,7 @@ init_mb_mgr_sse_t2_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_sse; state->keyexp_192 = aes_keyexp_192_sse; @@ -375,12 +377,12 @@ init_mb_mgr_sse_t2_internal(IMB_MGR *state, const int reset_mgrs) state->xcbc_keyexp = aes_xcbc_expand_key_sse; state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_sse; - state->sha1 = sha1_sse; - state->sha224_one_block = sha224_one_block_sse; - state->sha224 = sha224_sse; - state->sha256_one_block = sha256_one_block_sse; - state->sha256 = sha256_sse; + state->sha1_one_block = sha1_one_block_sse_shani; + state->sha1 = sha1_sse_shani; + state->sha224_one_block = sha224_one_block_sse_shani; + state->sha224 = sha224_sse_shani; + state->sha256_one_block = sha256_one_block_sse_shani; + state->sha256 = sha256_sse_shani; state->sha384_one_block = sha384_one_block_sse; state->sha384 = sha384_sse; state->sha512_one_block = sha512_one_block_sse; @@ -388,6 +390,7 @@ init_mb_mgr_sse_t2_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_sse; state->aes128_cfb_one = aes_cfb_128_one_sse; + state->aes256_cfb_one = aes_cfb_256_one_sse; state->eea3_1_buffer = zuc_eea3_1_buffer_sse; state->eea3_4_buffer = zuc_eea3_4_buffer_sse; @@ -483,6 +486,9 @@ init_mb_mgr_sse_t2_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_sse; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_sse; } #include "mb_mgr_code.h" diff --git a/lib/sse_t2/sha1_ni_one_block_sse.asm b/lib/sse_t2/sha1_ni_one_block_sse.asm new file mode 100644 index 0000000000000000000000000000000000000000..ef9e9093efcd43317e41c160758812d489d3e4d9 --- /dev/null +++ b/lib/sse_t2/sha1_ni_one_block_sse.asm @@ -0,0 +1,291 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.inc" +%ifdef LINUX +%define INP rdi ; 1st arg +%define CTX rsi ; 2nd arg +%define REG3 edx +%define REG4 ecx +%else +%define INP rcx ; 1st arg +%define CTX rdx ; 2nd arg +%define REG3 edi +%define REG4 esi +%endif + +struc frame +.ABCD_SAVE reso 1 +.E_SAVE reso 1 +.XMM_SAVE reso 3 +.align resq 1 +endstruc + +%define ABCD xmm0 +%define E0 xmm1 ; Need two E's b/c they ping pong +%define E1 xmm2 +%define MSG0 xmm3 +%define MSG1 xmm4 +%define MSG2 xmm5 +%define MSG3 xmm6 + +%define SHUF_MASK xmm14 +%define E_MASK xmm15 + +mksection .rodata +default rel +align 64 +PSHUFFLE_BYTE_FLIP_MASK: ;ddq 0x000102030405060708090a0b0c0d0e0f + dq 0x08090a0b0c0d0e0f, 0x0001020304050607 +UPPER_WORD_MASK: ;ddq 0xFFFFFFFF000000000000000000000000 + dq 0x0000000000000000, 0xFFFFFFFF00000000 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha1_ni_block_sse(void *input_data, UINT32 digest[5]) +;; arg 1 : (in) pointer to one block of data +;; arg 2 : (in/out) pointer to read/write digest + +mksection .text +MKGLOBAL(sha1_ni_block_sse,function,internal) +align 32 +sha1_ni_block_sse: + sub rsp, frame_size + +%ifndef LINUX + movdqa [rsp + frame.XMM_SAVE], xmm6 + movdqa [rsp + frame.XMM_SAVE + 16], xmm14 + movdqa [rsp + frame.XMM_SAVE + 16*2], xmm15 +%endif + + ;; load initial digest + movdqu ABCD, [CTX] + pxor E0, E0 + pinsrd E0, [CTX + 16], 3 + pshufd ABCD, ABCD, 0x1B + + movdqa SHUF_MASK, [rel PSHUFFLE_BYTE_FLIP_MASK] + movdqa E_MASK, [rel UPPER_WORD_MASK] + + ;; Copy digests + movdqa [rsp + frame.ABCD_SAVE], ABCD + movdqa [rsp + frame.E_SAVE], E0 + + ;; Only needed if not using sha1nexte for rounds 0-3 + pand E0, E_MASK + + ;; Rounds 0-3 + movdqu MSG0, [INP + 0*16] + pshufb MSG0, SHUF_MASK + paddd E0, MSG0 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + + ;; Rounds 4-7 + movdqu MSG1, [INP + 1*16] + pshufb MSG1, SHUF_MASK + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG0, MSG1 + + ;; Rounds 8-11 + movdqu MSG2, [INP + 2*16] + pshufb MSG2, SHUF_MASK + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 12-15 + movdqu MSG3, [INP + 3*16] + pshufb MSG3, SHUF_MASK + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 16-19 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 20-23 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + + ;; Rounds 24-27 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 28-31 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 32-35 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 36-39 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + + ;; Rounds 40-43 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 44-47 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 48-51 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 52-55 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + + ;; Rounds 56-59 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 60-63 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 3 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 64-67 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 3 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 68-71 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 3 + pxor MSG3, MSG1 + + ;; Rounds 72-75 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 3 + + ;; Rounds 76-79 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 3 + + ;; Need to rotate E left by 30 + movdqa E1, E0 + pslld E0, 30 + psrld E1, 2 + pxor E0, E1 + + paddd ABCD, [rsp + frame.ABCD_SAVE] + paddd E0, [rsp + frame.E_SAVE] + + ;; write out digests + pshufd ABCD, ABCD, 0x1B + movdqu [CTX], ABCD + pextrd [CTX + 16], E0, 3 + + ;; Clear stack frame (4*16 bytes) +%ifdef SAFE_DATA + pxor MSG0, MSG0 + pxor MSG1, MSG1 + pxor MSG2, MSG2 + pxor MSG3, MSG3 + + movdqa [rsp + frame.ABCD_SAVE], MSG0 + movdqa [rsp + frame.E_SAVE], MSG0 +%endif + +%ifndef LINUX + movdqa xmm6, [rsp + frame.XMM_SAVE] + movdqa xmm14, [rsp + frame.XMM_SAVE + 16] + movdqa xmm15, [rsp + frame.XMM_SAVE + 16*2] +%endif + add rsp, frame_size + + ret + +mksection stack-noexec diff --git a/lib/sse_t2/sha1_ni_x1_sse.asm b/lib/sse_t2/sha1_ni_x1_sse.asm index 8022746b91767bd79504f6848af9e3821fe59726..c3d1581d9b55181aee7cdc88d44228ff074eeefc 100644 --- a/lib/sse_t2/sha1_ni_x1_sse.asm +++ b/lib/sse_t2/sha1_ni_x1_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -38,11 +38,11 @@ ;; ;; Linux/Windows clobbers: xmm0 - xmm15 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/clear_regs.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/dbgprint.inc" +%include "include/clear_regs.inc" +%include "include/mb_mgr_datastruct.inc" %ifdef LINUX %define arg1 rdi @@ -64,9 +64,6 @@ struc frame .ABCD_SAVE reso 1 .E_SAVE reso 1 -.ABCD_SAVEb reso 1 -.E_SAVEb reso 1 -.XMM_SAVE reso 3 .align resq 1 endstruc @@ -104,10 +101,6 @@ align 32 sha1_ni_x1: sub rsp, frame_size - movdqa [rsp + frame.XMM_SAVE], xmm6 - movdqa [rsp + frame.XMM_SAVE + 16], xmm14 - movdqa [rsp + frame.XMM_SAVE + 16*2], xmm15 - shl NUM_BLKS, 6 ; convert to bytes jz done_hash @@ -323,13 +316,8 @@ done_hash: movdqa [rsp + 0*16], MSG0 movdqa [rsp + 1*16], MSG0 - movdqa [rsp + 2*16], MSG0 - movdqa [rsp + 3*16], MSG0 %endif - movdqa xmm6, [rsp + frame.XMM_SAVE] - movdqa xmm14, [rsp + frame.XMM_SAVE + 16] - movdqa xmm15, [rsp + frame.XMM_SAVE + 16*2] add rsp, frame_size ret diff --git a/lib/sse_t2/sha1_ni_x2_sse.asm b/lib/sse_t2/sha1_ni_x2_sse.asm index 27a61bbcb54dd01c2a4cb1770ea075cb36d6d378..f69ddee4f3b9249c37b3333dbee54f9fb4a84f67 100644 --- a/lib/sse_t2/sha1_ni_x2_sse.asm +++ b/lib/sse_t2/sha1_ni_x2_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -38,11 +38,11 @@ ;; ;; Linux/Windows clobbers: xmm0 - xmm15 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/clear_regs.asm" -%include "include/mb_mgr_datastruct.asm" +%include "include/dbgprint.inc" +%include "include/clear_regs.inc" +%include "include/mb_mgr_datastruct.inc" %ifdef LINUX %define arg1 rdi diff --git a/lib/sse_t2/sha256_ni_one_block_sse.asm b/lib/sse_t2/sha256_ni_one_block_sse.asm new file mode 100644 index 0000000000000000000000000000000000000000..9d1001604dc3b54ad0ff0a7238b0baeddc3cf541 --- /dev/null +++ b/lib/sse_t2/sha256_ni_one_block_sse.asm @@ -0,0 +1,323 @@ +; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.inc" +%include "include/clear_regs.inc" + +; resdq = res0 => 16 bytes +struc frame +.ABEF_SAVE reso 1 +.CDGH_SAVE reso 1 +.XMM_SAVE reso 3 +.align resq 1 +endstruc + +%ifdef LINUX +%define INP rdi ; 1st arg +%define CTX rsi ; 2nd arg +%define REG3 edx +%define REG4 ecx +%else +%define INP rcx ; 1st arg +%define CTX rdx ; 2nd arg +%define REG3 edi +%define REG4 esi +%endif + +;; MSG MUST be xmm0 (implicit argument) +%define MSG xmm0 +%define STATE0 xmm1 +%define STATE1 xmm2 +%define MSGTMP0 xmm3 +%define MSGTMP1 xmm4 +%define MSGTMP2 xmm5 +%define MSGTMP3 xmm6 +%define MSGTMP4 xmm7 +%define MSGTMP xmm14 +%define SHUF_MASK xmm15 + +mksection .rodata +default rel + +extern K256 + +align 64 +PSHUFFLE_BYTE_FLIP_MASK: + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + +mksection .text +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha256_ni_block_sse(void *input_data, UINT32 digest[8]) +;; arg 1 : (in) pointer to one block of data +;; arg 2 : (in/out) pointer to read/write digest + +MKGLOBAL(sha256_ni_block_sse,function,internal) +align 32 +sha256_ni_block_sse: + sub rsp, frame_size + +%ifndef LINUX + movdqa [rsp + frame.XMM_SAVE], xmm6 + movdqa [rsp + frame.XMM_SAVE + 16], xmm14 + movdqa [rsp + frame.XMM_SAVE + 16*2], xmm15 +%endif + + ;; load initial digest + ;; Probably need to reorder these appropriately + ;; DCBA, HGFE -> ABEF, CDGH + movdqu STATE0, [CTX] + movdqu STATE1, [CTX + 16] + + pshufd STATE0, STATE0, 0xB1 ; CDAB + pshufd STATE1, STATE1, 0x1B ; EFGH + movdqa MSGTMP4, STATE0 + palignr STATE0, STATE1, 8 ; ABEF + pblendw STATE1, MSGTMP4, 0xF0 ; CDGH + + movdqa SHUF_MASK, [rel PSHUFFLE_BYTE_FLIP_MASK] + + ;; Save digests + movdqa [rsp + frame.ABEF_SAVE], STATE0 + movdqa [rsp + frame.CDGH_SAVE], STATE1 + + ;; Rounds 0-3 + movdqu MSG, [INP + 0*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP0, MSG + paddd MSG, [rel K256 + 0*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + ;; Rounds 4-7 + movdqu MSG, [INP + 1*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP1, MSG + paddd MSG, [rel K256 + 1*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP0, MSGTMP1 + + ;; Rounds 8-11 + movdqu MSG, [INP + 2*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP2, MSG + paddd MSG, [rel K256 + 2*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP1, MSGTMP2 + + ;; Rounds 12-15 + movdqu MSG, [INP + 3*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP3, MSG + paddd MSG, [rel K256 + 3*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP3 + palignr MSGTMP, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP2, MSGTMP3 + + ;; Rounds 16-19 + movdqa MSG, MSGTMP0 + paddd MSG, [rel K256 + 4*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP0 + palignr MSGTMP, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP3, MSGTMP0 + + ;; Rounds 20-23 + movdqa MSG, MSGTMP1 + paddd MSG, [rel K256 + 5*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP1 + palignr MSGTMP, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP0, MSGTMP1 + + ;; Rounds 24-27 + movdqa MSG, MSGTMP2 + paddd MSG, [rel K256 + 6*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP2 + palignr MSGTMP, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP1, MSGTMP2 + + ;; Rounds 28-31 + movdqa MSG, MSGTMP3 + paddd MSG, [rel K256 + 7*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP3 + palignr MSGTMP, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP2, MSGTMP3 + + ;; Rounds 32-35 + movdqa MSG, MSGTMP0 + paddd MSG, [rel K256 + 8*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP0 + palignr MSGTMP, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP3, MSGTMP0 + + ;; Rounds 36-39 + movdqa MSG, MSGTMP1 + paddd MSG, [rel K256 + 9*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP1 + palignr MSGTMP, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP0, MSGTMP1 + + ;; Rounds 40-43 + movdqa MSG, MSGTMP2 + paddd MSG, [rel K256 + 10*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP2 + palignr MSGTMP, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP1, MSGTMP2 + + ;; Rounds 44-47 + movdqa MSG, MSGTMP3 + paddd MSG, [rel K256 + 11*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP3 + palignr MSGTMP, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP2, MSGTMP3 + + ;; Rounds 48-51 + movdqa MSG, MSGTMP0 + paddd MSG, [rel K256 + 12*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP0 + palignr MSGTMP, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP3, MSGTMP0 + + ;; Rounds 52-55 + movdqa MSG, MSGTMP1 + paddd MSG, [rel K256 + 13*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP1 + palignr MSGTMP, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + ;; Rounds 56-59 + movdqa MSG, MSGTMP2 + paddd MSG, [rel K256 + 14*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP2 + palignr MSGTMP, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + ;; Rounds 60-63 + movdqa MSG, MSGTMP3 + paddd MSG, [rel K256 + 15*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + paddd STATE0, [rsp + frame.ABEF_SAVE] + paddd STATE1, [rsp + frame.CDGH_SAVE] + + ; Reorder for writeback + pshufd STATE0, STATE0, 0x1B ; FEBA + pshufd STATE1, STATE1, 0xB1 ; DCHG + movdqa MSGTMP4, STATE0 + pblendw STATE0, STATE1, 0xF0 ; DCBA + palignr STATE1, MSGTMP4, 8 ; HGFE + + ;; update digests + movdqu [CTX], STATE0 + movdqu [CTX + 16], STATE1 + + ;; Clear stack frame (2*16 bytes) +%ifdef SAFE_DATA + pxor MSGTMP0, MSGTMP0 + pxor MSGTMP1, MSGTMP1 + pxor MSGTMP2, MSGTMP2 + pxor MSGTMP3, MSGTMP3 + pxor MSGTMP4, MSGTMP4 + pxor MSGTMP, MSGTMP + + movdqa [rsp + frame.ABEF_SAVE], MSGTMP0 + movdqa [rsp + frame.CDGH_SAVE], MSGTMP0 +%endif + +%ifndef LINUX + movdqa xmm6, [rsp + frame.XMM_SAVE] + movdqa xmm14, [rsp + frame.XMM_SAVE + 16] + movdqa xmm15, [rsp + frame.XMM_SAVE + 16*2] +%endif + add rsp, frame_size + ret + +mksection stack-noexec diff --git a/lib/sse_t2/sha256_ni_x1_sse.asm b/lib/sse_t2/sha256_ni_x1_sse.asm index b6a026476071a29f6bb13167adac7e3d6291031c..46309a1568692f202227fc1f57510bffdcaa46bb 100644 --- a/lib/sse_t2/sha256_ni_x1_sse.asm +++ b/lib/sse_t2/sha256_ni_x1_sse.asm @@ -1,5 +1,5 @@ ; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -38,10 +38,10 @@ ;; ;; Linux/Windows clobbers: xmm0 - xmm15 -%include "include/os.asm" +%include "include/os.inc" %include "include/cet.inc" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" ; resdq = res0 => 16 bytes struc frame @@ -184,7 +184,7 @@ sha256_ni_x1: shl lane, 5 movdqu STATE0, [args + lane] movdqu STATE1, [args + lane + 16] - + pshufd STATE0, STATE0, 0xB1 ; CDAB pshufd STATE1, STATE1, 0x1B ; EFGH movdqa MSGTMP4, STATE0 @@ -403,7 +403,7 @@ sha256_ni_x1: done_hash: - ;; Clear stack frame (4*16 bytes) + ;; Clear stack frame (2*16 bytes) %ifdef SAFE_DATA clear_all_xmms_sse_asm movdqa [rsp + frame.ABEF_SAVE], xmm0 diff --git a/lib/sse_t2/sha256_ni_x2_sse.asm b/lib/sse_t2/sha256_ni_x2_sse.asm index c9aafac196397e80bdb4f717ddf0666223c3f4c6..0c5cdacd4ac259c594ab286496cbd71b0a8ca886 100644 --- a/lib/sse_t2/sha256_ni_x2_sse.asm +++ b/lib/sse_t2/sha256_ni_x2_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -38,11 +38,11 @@ ;; ;; Linux/Windows clobbers: xmm0 - xmm15 -%include "include/os.asm" +%include "include/os.inc" ;%define DO_DBGPRINT -%include "include/dbgprint.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/dbgprint.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" ; resdq = res0 => 16 bytes struc frame diff --git a/lib/sse_t2/sha_ni_mb_sse.c b/lib/sse_t2/sha_ni_mb_sse.c index 5a6cbec8caa87f9ed35d6f7a930c4af69a39e946..b872a5abf49fc956b0e62e9a20f1f4812d2edd38 100644 --- a/lib/sse_t2/sha_ni_mb_sse.c +++ b/lib/sse_t2/sha_ni_mb_sse.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -89,4 +89,4 @@ IMB_JOB *flush_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) return submit_flush_job_sha_256(state, job, 2, 0, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, call_sha256_ni_x2_sse_from_c, 1); -} \ No newline at end of file +} diff --git a/lib/sse_t2/sha_ni_sse.c b/lib/sse_t2/sha_ni_sse.c new file mode 100644 index 0000000000000000000000000000000000000000..b3efb20e83949baac492de413b9df1d44a827215 --- /dev/null +++ b/lib/sse_t2/sha_ni_sse.c @@ -0,0 +1,82 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_generic.h" +#include "include/arch_sse_type2.h" + +/* ========================================================================== */ +/* One block SHA1 computation for IPAD / OPAD usage only */ + +void sha1_one_block_sse_shani(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_SSE_SHANI, 1 /* SHA1 */); +} + +/* ========================================================================== */ +/* One block SHA224 computation for IPAD / OPAD usage only */ +void sha224_one_block_sse_shani(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_SSE_SHANI, 224 /* SHA224 */); +} + +/* ========================================================================== */ +/* ========================================================================== */ +/* One block SHA256 computation for IPAD / OPAD usage only */ +void sha256_one_block_sse_shani(const void *data, void *digest) +{ + sha_generic_1block(data, digest, ARCH_SSE_SHANI, 256 /* SHA256 */); +} + +/* ========================================================================== */ +/* + * SHA1 API for use in HMAC-SHA1 when key is longer than the block size + */ + +void sha1_sse_shani(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_SSE_SHANI, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE); +} + +/* ========================================================================== */ +/* + * SHA224 API for use in HMAC-SHA224 when key is longer than the block size + */ +void sha224_sse_shani(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_SSE_SHANI, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE); +} +/* ========================================================================== */ +/* + * SHA256 API for use in HMAC-SHA256 when key is longer than the block size + */ +void sha256_sse_shani(const void *data, const uint64_t length, void *digest) +{ + sha_generic(data, length, digest, ARCH_SSE_SHANI, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE); +} diff --git a/lib/sse_t3/aes128_cbc_dec_by8_sse.asm b/lib/sse_t3/aes128_cbc_dec_by8_sse.asm index 862edf5a321ec46293b43219fdd4b67c371a4c81..d9ca933e313cb1435da69ee4ba7cba490f3867e2 100644 --- a/lib/sse_t3/aes128_cbc_dec_by8_sse.asm +++ b/lib/sse_t3/aes128_cbc_dec_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -41,8 +41,8 @@ ; arg 4: OUT: pointer to output (plain text) ; arg 5: LEN: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_DEC_128 %define AES_CBC_DEC_128 aes_cbc_dec_128_by8_sse diff --git a/lib/sse_t3/aes128_cbc_enc_x8_sse.asm b/lib/sse_t3/aes128_cbc_enc_x8_sse.asm index 2888cf236aded79fc75c0cbcbeca41d2bd4048f4..374af50044f3be3e964688cf5c0f7a2d011a1e69 100644 --- a/lib/sse_t3/aes128_cbc_enc_x8_sse.asm +++ b/lib/sse_t3/aes128_cbc_enc_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,9 +29,9 @@ ;;; processes 8 buffers at a time, single data structure as input ;;; Updates In and Out pointers at end -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" %macro PXOR2 2 movdqu XTMP, %2 diff --git a/lib/sse_t3/aes128_cbc_mac_x8_sse.asm b/lib/sse_t3/aes128_cbc_mac_x8_sse.asm index 119a60d50d821421e2391fcb003dff037335e22b..9f09c1b9e1d8eafc330869232051afe0209999bc 100644 --- a/lib/sse_t3/aes128_cbc_mac_x8_sse.asm +++ b/lib/sse_t3/aes128_cbc_mac_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2017-2022, Intel Corporation +;; Copyright (c) 2017-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/aes128_ecb_by8_sse.asm b/lib/sse_t3/aes128_ecb_by8_sse.asm index 9c6a5aedc70b7cc7022fd63860b040a75609b410..caf72a4b4d43a602f126bcf02d8f234e07f87658 100644 --- a/lib/sse_t3/aes128_ecb_by8_sse.asm +++ b/lib/sse_t3/aes128_ecb_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -27,9 +27,9 @@ ; routine to do AES ECB 128 encrypt/decrypt on 16n bytes doing AES by 8 -%include "include/os.asm" -%include "include/clear_regs.asm" -%include "include/aes_common.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/aes_common.inc" %ifdef LINUX %define IN rdi diff --git a/lib/sse_t3/aes192_cbc_dec_by8_sse.asm b/lib/sse_t3/aes192_cbc_dec_by8_sse.asm index c04f546b74ac5d4da40ad8ff8e9ebad2c2bee6f2..cba7737454315b32b803143c24e8d9879f87a807 100644 --- a/lib/sse_t3/aes192_cbc_dec_by8_sse.asm +++ b/lib/sse_t3/aes192_cbc_dec_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -41,8 +41,8 @@ ; arg 4: OUT: pointer to output (plain text) ; arg 5: LEN: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_DEC_192 %define AES_CBC_DEC_192 aes_cbc_dec_192_by8_sse diff --git a/lib/sse_t3/aes192_cbc_enc_x8_sse.asm b/lib/sse_t3/aes192_cbc_enc_x8_sse.asm index 68bd9b7afefd85ebcff4c02c6f619f3aa2cdb6b6..a630b176790e2a67da83d62dd2793359361668f8 100644 --- a/lib/sse_t3/aes192_cbc_enc_x8_sse.asm +++ b/lib/sse_t3/aes192_cbc_enc_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,9 +29,9 @@ ;; clobbers all registers except for ARG1 and rbp -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" %macro PXOR2 2 movdqu XTMP, %2 diff --git a/lib/sse_t3/aes192_ecb_by8_sse.asm b/lib/sse_t3/aes192_ecb_by8_sse.asm index 8bc905b4c001dd304adb8dce4e8bd2244595b4b5..49e0b267df359347a1b965c9f5bf568bc31960e0 100644 --- a/lib/sse_t3/aes192_ecb_by8_sse.asm +++ b/lib/sse_t3/aes192_ecb_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,5 +29,5 @@ %define AES_ECB_NROUNDS 12 -%include "include/os.asm" +%include "include/os.inc" %include "sse_t3/aes128_ecb_by8_sse.asm" diff --git a/lib/sse_t3/aes256_cbc_dec_by8_sse.asm b/lib/sse_t3/aes256_cbc_dec_by8_sse.asm index bc0fb718e20c4ae06b7b8c2c25385d5f72b52683..cee35d2226d557edc6aa4d4185800f2f013c8858 100644 --- a/lib/sse_t3/aes256_cbc_dec_by8_sse.asm +++ b/lib/sse_t3/aes256_cbc_dec_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -42,8 +42,8 @@ ; arg 5: sp: length in bytes (multiple of 16) ; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %ifndef AES_CBC_DEC_256 %define AES_CBC_DEC_256 aes_cbc_dec_256_by8_sse diff --git a/lib/sse_t3/aes256_cbc_enc_x8_sse.asm b/lib/sse_t3/aes256_cbc_enc_x8_sse.asm index 851a4a759114f90c5808c20947c342c6d90e1026..964798e612554de090622927206a198e0f8d6ee2 100644 --- a/lib/sse_t3/aes256_cbc_enc_x8_sse.asm +++ b/lib/sse_t3/aes256_cbc_enc_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,9 +29,9 @@ ;; clobbers all registers except for ARG1 and rbp -%include "include/os.asm" -%include "include/mb_mgr_datastruct.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/mb_mgr_datastruct.inc" +%include "include/clear_regs.inc" %macro PXOR2 2 movdqu XTMP, %2 diff --git a/lib/sse_t3/aes256_cbc_mac_x8_sse.asm b/lib/sse_t3/aes256_cbc_mac_x8_sse.asm index 75d861a0bacdf395ab6d93a329265d92ba967416..289e4081a7d56d3d0b043e140e952f7a2d69e60d 100644 --- a/lib/sse_t3/aes256_cbc_mac_x8_sse.asm +++ b/lib/sse_t3/aes256_cbc_mac_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/aes256_ecb_by8_sse.asm b/lib/sse_t3/aes256_ecb_by8_sse.asm index d40d245d8560816ab06f2c80249fd2e916611183..56058434a053388ab577a4611f1d449ec573e354 100644 --- a/lib/sse_t3/aes256_ecb_by8_sse.asm +++ b/lib/sse_t3/aes256_ecb_by8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2022, Intel Corporation +;; Copyright (c) 2022-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -29,5 +29,5 @@ %define AES_ECB_NROUNDS 14 -%include "include/os.asm" +%include "include/os.inc" %include "sse_t3/aes128_ecb_by8_sse.asm" diff --git a/lib/sse_t3/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm index 0fa6e860ee63875075b7e61a26d12b198dc41999..2cffb8e9f0ca6c43786c4eb41fe36719940553bf 100644 --- a/lib/sse_t3/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm index 54244bded10f0c4230da1c10c0df82a1f6a02a5b..0d792581a7f36c4bd49364b20fc39de579bf9e4c 100644 --- a/lib/sse_t3/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm index 4c74848dfdc96ad76492606e79aa7d5d69a996e7..0203de7605a49f23f7cff6b5b3d75ae5e54e9189 100644 --- a/lib/sse_t3/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm index e3eebcac953b08579455437bdf0eeb40bd054de0..83fd0528f3598c67649978856da00b79a975528b 100644 --- a/lib/sse_t3/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm index 657b6c6bbd789ae37281dc4433abc2187840c1d3..de23167ff071feadc095d486776699cda0394bd7 100644 --- a/lib/sse_t3/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm b/lib/sse_t3/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm index 28ca090aaf7f9bf73313da2732468444f61fbaa9..a7094a95d6f7d45ef12946cdafe739d28b2b9a32 100644 --- a/lib/sse_t3/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm index 33d72ab742ad1773e6903537c152d01221def152..06bad243eb44dfeccc6c5e74295d29f1b9a7016c 100644 --- a/lib/sse_t3/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm index fd2d2a768d4b74cc68faf25b35dfe189b08a85bc..9ffa41789c187bb643e89c9f71a4803c344e2cc6 100644 --- a/lib/sse_t3/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm index c540763789a9a1e1c705a40ab72f7d5f80ce2578..17f9f954c62f8bed626ae79970d0426ec32b2e7c 100644 --- a/lib/sse_t3/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm index 62aeb6e861d763727804f597a12c6fc1cd5925fa..907235833b368854c8768232c5033a55c12482d0 100644 --- a/lib/sse_t3/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/mb_mgr_sse_t3.c b/lib/sse_t3/mb_mgr_sse_t3.c index c0a244244cb212d2eaa5a2ddab083515c6792dc1..50ef1465227e30525c6c18c119f86a3b566ca4a0 100644 --- a/lib/sse_t3/mb_mgr_sse_t3.c +++ b/lib/sse_t3/mb_mgr_sse_t3.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -71,6 +71,7 @@ #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_t3 #define SUBMIT_HASH_BURST submit_hash_burst_sse_t3 #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_t3 +#define SET_SUITE_ID_FN set_suite_id_sse_t3 /* Hash */ #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE_T3 @@ -365,6 +366,7 @@ init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; state->submit_hash_burst = SUBMIT_HASH_BURST; state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->set_suite_id = SET_SUITE_ID_FN; state->keyexp_128 = aes_keyexp_128_sse; state->keyexp_192 = aes_keyexp_192_sse; @@ -376,12 +378,12 @@ init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs) state->xcbc_keyexp = aes_xcbc_expand_key_sse; state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_sse; - state->sha1 = sha1_sse; - state->sha224_one_block = sha224_one_block_sse; - state->sha224 = sha224_sse; - state->sha256_one_block = sha256_one_block_sse; - state->sha256 = sha256_sse; + state->sha1_one_block = sha1_one_block_sse_shani; + state->sha1 = sha1_sse_shani; + state->sha224_one_block = sha224_one_block_sse_shani; + state->sha224 = sha224_sse_shani; + state->sha256_one_block = sha256_one_block_sse_shani; + state->sha256 = sha256_sse_shani; state->sha384_one_block = sha384_one_block_sse; state->sha384 = sha384_sse; state->sha512_one_block = sha512_one_block_sse; @@ -389,6 +391,7 @@ init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs) state->md5_one_block = md5_one_block_sse; state->aes128_cfb_one = aes_cfb_128_one_sse; + state->aes256_cfb_one = aes_cfb_256_one_sse; state->eea3_1_buffer = zuc_eea3_1_buffer_sse; state->eea3_4_buffer = zuc_eea3_4_buffer_gfni_sse; @@ -484,6 +487,9 @@ init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; + + state->aes_ecb_128_quic = aes_ecb_quic_enc_128_sse; + state->aes_ecb_256_quic = aes_ecb_quic_enc_256_sse; } #include "mb_mgr_code.h" diff --git a/lib/sse_t3/mb_mgr_zuc_submit_flush_gfni_sse.asm b/lib/sse_t3/mb_mgr_zuc_submit_flush_gfni_sse.asm index 77d26c1d90273b27edd34d046d083842706fac35..d2d49b692dccdd0ff555e2fbe34ea55d2a310b56 100644 --- a/lib/sse_t3/mb_mgr_zuc_submit_flush_gfni_sse.asm +++ b/lib/sse_t3/mb_mgr_zuc_submit_flush_gfni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/sse_t3/zuc_x4_gfni_sse.asm b/lib/sse_t3/zuc_x4_gfni_sse.asm index 6f269bfe1aba856057c23efa50655c1f936709fa..8b5848fa5ebe84e87db0ba6c2a51f5ed22d14a85 100644 --- a/lib/sse_t3/zuc_x4_gfni_sse.asm +++ b/lib/sse_t3/zuc_x4_gfni_sse.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: diff --git a/lib/win_x64.mak b/lib/win_x64.mak index d0e5df95cd6b766348dfc3ff42a72c55759416c6..f21b8e546522b347e9bdde93cf5525365c487d8a 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -1,5 +1,5 @@ # -# Copyright (c) 2017-2022, Intel Corporation +# Copyright (c) 2017-2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -112,8 +112,8 @@ DAFLAGS = $(DAFLAGS) -DAESNI_EMU CC = cl -CFLAGS_ALL = $(EXTRA_CFLAGS) /DNO_COMPAT_IMB_API_053 /I. /Iinclude /Ino-aesni \ - /nologo /Y- /W3 /WX- /Gm- /fp:precise /EHsc /Z7 +CFLAGS_ALL = $(EXTRA_CFLAGS) /I. /Iinclude /Ino-aesni \ + /nologo /Y- /W3 /WX- /Gm- /fp:precise /EHsc /Z7 /std:c11 CFLAGS = $(CFLAGS_ALL) $(OPT) $(DCFLAGS) CFLAGS_NO_SIMD = $(CFLAGS_ALL) /Od $(DCFLAGS) @@ -161,10 +161,13 @@ lib_objs1 = \ $(OBJ_DIR)\aes128_ecb_by8_avx.obj \ $(OBJ_DIR)\aes192_ecb_by8_avx.obj \ $(OBJ_DIR)\aes256_ecb_by8_avx.obj \ + $(OBJ_DIR)\aes_ecb_quic_x8_sse.obj \ + $(OBJ_DIR)\aes_ecb_quic_x8_avx.obj \ $(OBJ_DIR)\aes128_ecb_vaes_avx2.obj \ $(OBJ_DIR)\aes192_ecb_vaes_avx2.obj \ $(OBJ_DIR)\aes256_ecb_vaes_avx2.obj \ $(OBJ_DIR)\aes_ecb_vaes_avx512.obj \ + $(OBJ_DIR)\aes_ecb_quic_vaes_avx512.obj \ $(OBJ_DIR)\pon_by8_sse.obj \ $(OBJ_DIR)\aes128_cntr_by8_sse.obj \ $(OBJ_DIR)\pon_by8_avx.obj \ @@ -216,6 +219,8 @@ lib_objs1 = \ $(OBJ_DIR)\md5_x4x2_sse.obj \ $(OBJ_DIR)\md5_x8x2_avx2.obj \ $(OBJ_DIR)\save_xmms.obj \ + $(OBJ_DIR)\mbcpuid.obj \ + $(OBJ_DIR)\atomic.obj \ $(OBJ_DIR)\clear_regs_mem_fns.obj \ $(OBJ_DIR)\sha1_x4_avx.obj \ $(OBJ_DIR)\sha1_x4_sse.obj \ @@ -223,6 +228,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha1_ni_x1_sse.obj \ $(OBJ_DIR)\sha1_one_block_avx.obj \ $(OBJ_DIR)\sha1_one_block_sse.obj \ + $(OBJ_DIR)\sha1_ni_one_block_sse.obj \ $(OBJ_DIR)\sha1_x8_avx2.obj \ $(OBJ_DIR)\sha1_x16_avx512.obj \ $(OBJ_DIR)\sha224_one_block_avx.obj \ @@ -230,6 +236,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha256_oct_avx2.obj \ $(OBJ_DIR)\sha256_one_block_avx.obj \ $(OBJ_DIR)\sha256_one_block_sse.obj \ + $(OBJ_DIR)\sha256_ni_one_block_sse.obj \ $(OBJ_DIR)\sha256_ni_x2_sse.obj \ $(OBJ_DIR)\sha256_ni_x1_sse.obj \ $(OBJ_DIR)\sha256_x16_avx512.obj \ @@ -255,11 +262,11 @@ lib_objs1 = \ $(OBJ_DIR)\zuc_x4_gfni_sse.obj \ $(OBJ_DIR)\zuc_x4_avx.obj \ $(OBJ_DIR)\zuc_x8_avx2.obj \ + $(OBJ_DIR)\zuc_x8_gfni_avx2.obj \ $(OBJ_DIR)\zuc_x16_avx512.obj \ $(OBJ_DIR)\zuc_x16_vaes_avx512.obj \ $(OBJ_DIR)\zuc_iv.obj \ $(OBJ_DIR)\snow3g_sse.obj \ - $(OBJ_DIR)\snow3g_uea2_by4_sse.obj \ $(OBJ_DIR)\snow3g_uia2_by4_sse.obj \ $(OBJ_DIR)\snow3g_avx.obj \ $(OBJ_DIR)\snow3g_avx2.obj \ @@ -275,7 +282,10 @@ lib_objs1 = \ $(OBJ_DIR)\aes_xcbc_expand_key.obj \ $(OBJ_DIR)\md5_one_block.obj \ $(OBJ_DIR)\sha_sse.obj \ + $(OBJ_DIR)\sha_ni_sse.obj \ $(OBJ_DIR)\sha_avx.obj \ + $(OBJ_DIR)\sha_avx2.obj \ + $(OBJ_DIR)\sha_avx512.obj \ $(OBJ_DIR)\sha_mb_sse.obj \ $(OBJ_DIR)\sha_ni_mb_sse.obj \ $(OBJ_DIR)\sha_mb_avx.obj \ @@ -351,7 +361,11 @@ lib_objs1 = \ $(OBJ_DIR)\memcpy_sse.obj \ $(OBJ_DIR)\memcpy_avx.obj \ $(OBJ_DIR)\ooo_mgr_reset.obj \ - $(OBJ_DIR)\self_test.obj + $(OBJ_DIR)\self_test.obj \ + $(OBJ_DIR)\quic_aes_gcm.obj \ + $(OBJ_DIR)\quic_hp_aes_ecb.obj \ + $(OBJ_DIR)\hmac_ipad_opad.obj \ + $(OBJ_DIR)\cipher_suite_id.obj lib_objs2 = \ $(OBJ_DIR)\mb_mgr_aes192_cbc_enc_flush_avx.obj \ @@ -444,6 +458,7 @@ lib_objs2 = \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_gfni_sse.obj \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_avx.obj \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_avx2.obj \ + $(OBJ_DIR)\mb_mgr_zuc_submit_flush_gfni_avx2.obj \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_avx512.obj \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_gfni_avx512.obj \ $(OBJ_DIR)\mb_mgr_avx.obj \ @@ -536,24 +551,21 @@ no_aesni_objs = \ gcm_objs = \ $(OBJ_DIR)\gcm.obj \ - $(OBJ_DIR)\aes128_gcm_by8_avx.obj \ $(OBJ_DIR)\aes128_gcm_by8_avx2.obj \ $(OBJ_DIR)\aes128_gcm_by8_avx512.obj \ - $(OBJ_DIR)\aes128_gcm_by48_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes128_gcm_by48_sgl_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes128_gmac_by48_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes192_gcm_by8_avx.obj \ + $(OBJ_DIR)\aes128_gcm_api_vaes_avx512.obj \ + $(OBJ_DIR)\aes128_gcm_sgl_api_vaes_avx512.obj \ + $(OBJ_DIR)\aes128_gmac_api_vaes_avx512.obj \ $(OBJ_DIR)\aes192_gcm_by8_avx2.obj \ $(OBJ_DIR)\aes192_gcm_by8_avx512.obj \ - $(OBJ_DIR)\aes192_gcm_by48_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes192_gcm_by48_sgl_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes192_gmac_by48_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes256_gcm_by8_avx.obj \ + $(OBJ_DIR)\aes192_gcm_api_vaes_avx512.obj \ + $(OBJ_DIR)\aes192_gcm_sgl_api_vaes_avx512.obj \ + $(OBJ_DIR)\aes192_gmac_api_vaes_avx512.obj \ $(OBJ_DIR)\aes256_gcm_by8_avx2.obj \ $(OBJ_DIR)\aes256_gcm_by8_avx512.obj \ - $(OBJ_DIR)\aes256_gcm_by48_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes256_gcm_by48_sgl_api_vaes_avx512.obj \ - $(OBJ_DIR)\aes256_gmac_by48_api_vaes_avx512.obj \ + $(OBJ_DIR)\aes256_gcm_api_vaes_avx512.obj \ + $(OBJ_DIR)\aes256_gcm_sgl_api_vaes_avx512.obj \ + $(OBJ_DIR)\aes256_gmac_api_vaes_avx512.obj \ $(OBJ_DIR)\gcm128_api_by8_sse.obj \ $(OBJ_DIR)\gcm128_sgl_api_by8_sse.obj \ $(OBJ_DIR)\gcm128_gmac_api_by8_sse.obj \ @@ -570,6 +582,11 @@ all_objs = $(lib_objs1) $(lib_objs2) $(gcm_objs) $(no_aesni_objs) all_objs = $(lib_objs1) $(lib_objs2) $(gcm_objs) !endif +!if "$(AVX_IFMA)" == "y" +all_objs = $(all_objs) $(OBJ_DIR)\mb_mgr_avx2_t3.obj $(OBJ_DIR)\poly_fma_avx2.obj +DCFLAGS = $(DCFLAGS) /DAVX_IFMA +!endif + all: $(LIB_DIR)\$(LIBNAME) $(DEPALL) $(LIB_DIR)\$(LIBNAME): $(all_objs) $(LIBBASE)_lnk.def @@ -593,13 +610,26 @@ $(LIB_DIR)\$(LIBNAME): $(all_objs) $(LIBBASE)_lnk.def @echo NOTE: $(SAFE_OPTIONS_MSG1) $(SAFE_OPTIONS_MSG2) !endif +STR_FILTER = "" +!if "$(AESNI_EMU)" != "y" +!if "$(AVX_IFMA)" != "y" +STR_FILTER = "_no_aesni _avx2_t3" +!else +STR_FILTER = "_no_aesni" +!endif +!else +!if "$(AVX_IFMA)" != "y" +STR_FILTER = "_avx2_t3" +!endif +!endif + $(all_objs): $(OBJ_DIR) $(LIB_DIR) $(LIBBASE)_lnk.def: $(LIBBASE).def -!if "$(AESNI_EMU)" == "y" +!if $(STR_FILTER) == "" copy /Y $(LIBBASE).def $(LIBBASE)_lnk.def !else - findstr /v _no_aesni $(LIBBASE).def > $(LIBBASE)_lnk.def + findstr /v $(STR_FILTER) $(LIBBASE).def > $(LIBBASE)_lnk.def !endif $(DEPALL): $(all_objs) @@ -661,6 +691,13 @@ $(DEPALL): $(all_objs) {avx2_t2\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< +{avx2_t3\}.c{$(OBJ_DIR)}.obj: + $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{avx2_t3\}.asm{$(OBJ_DIR)}.obj: + $(AS) -MD $@.dep -o $@ $(AFLAGS) $< + {avx512_t1\}.c{$(OBJ_DIR)}.obj: $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep diff --git a/lib/x86_64/aes128_ecbenc_x3.asm b/lib/x86_64/aes128_ecbenc_x3.asm index 01fbfa983bdd691c3267f0694386d60da8bd2822..0550b13b0ab367cde2c74f65466634b4711e7deb 100644 --- a/lib/x86_64/aes128_ecbenc_x3.asm +++ b/lib/x86_64/aes128_ecbenc_x3.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -32,10 +32,10 @@ ;void ; aes128_ecbenc_x3_avx(void *in, void *keys, void *out1, void *out2, void *out3); -%include "include/os.asm" +%include "include/os.inc" %define NO_AESNI_RENAME %include "include/aesni_emu.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %ifdef LINUX %define IN rdi ; arg 1 diff --git a/lib/x86_64/aes_cmac_subkey_gen.asm b/lib/x86_64/aes_cmac_subkey_gen.asm index 8324f9929f2b5ee047ae501e72faac6dfa3010f8..36f2c9a11cdb3c2f6404c0fd045aa5fd37d5e978 100644 --- a/lib/x86_64/aes_cmac_subkey_gen.asm +++ b/lib/x86_64/aes_cmac_subkey_gen.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" %define NO_AESNI_RENAME %include "include/aesni_emu.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" ;;; Routines to generate subkeys for AES-CMAC. diff --git a/lib/x86_64/aes_keyexp_128.asm b/lib/x86_64/aes_keyexp_128.asm index a6b93306aa0415bbd80c6d6e191a25686a6ce7d4..3298d7e9997b50f9d094c825c5937c4d608c00c7 100644 --- a/lib/x86_64/aes_keyexp_128.asm +++ b/lib/x86_64/aes_keyexp_128.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -26,10 +26,10 @@ ;; ; Routine to do AES key expansion -%include "include/os.asm" +%include "include/os.inc" %define NO_AESNI_RENAME %include "include/aesni_emu.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" %macro key_expansion_128_sse 0 diff --git a/lib/x86_64/aes_keyexp_192.asm b/lib/x86_64/aes_keyexp_192.asm index 0d68943f471361c63488ad1625c5d3b6e3861b64..fb8901820a3d0ba1ab15841e1b2a1ee5eb56fc11 100644 --- a/lib/x86_64/aes_keyexp_192.asm +++ b/lib/x86_64/aes_keyexp_192.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,10 +25,10 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" %define NO_AESNI_RENAME %include "include/aesni_emu.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" %ifdef LINUX diff --git a/lib/x86_64/aes_keyexp_256.asm b/lib/x86_64/aes_keyexp_256.asm index c4b58c8645a9b409f96f286218830e764e3775aa..562fcf321e6ea9698acde7acd998d255c98be271 100644 --- a/lib/x86_64/aes_keyexp_256.asm +++ b/lib/x86_64/aes_keyexp_256.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -26,10 +26,10 @@ ;; ; Routine to do AES key expansion -%include "include/os.asm" +%include "include/os.inc" %define NO_AESNI_RENAME %include "include/aesni_emu.inc" -%include "include/clear_regs.asm" +%include "include/clear_regs.inc" %include "include/cet.inc" %include "include/error.inc" ; Uses the f() function of the aeskeygenassist result diff --git a/lib/x86_64/aes_xcbc_expand_key.c b/lib/x86_64/aes_xcbc_expand_key.c index 1313f07d568a19920732707524046fff24c35015..7a9c17720df0a327c21de0dd659b63bd84dc8de0 100644 --- a/lib/x86_64/aes_xcbc_expand_key.c +++ b/lib/x86_64/aes_xcbc_expand_key.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 683c7635cb51cc04b76744f86530a59443993216..e4708a5a636c51e480c4dce0eac4a6af822673b0 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -27,7 +27,8 @@ #include #ifdef LINUX -#include /* posix_memalign() and free() */ +#include /* free() */ +#include /* memalign() */ #else #include /* _aligned_malloc() and aligned_free() */ #endif @@ -165,7 +166,8 @@ static void set_ooo_mgr_road_block(IMB_MGR *mgr) * IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, * currently SHANI is only available for SSE * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. - * IMB_FLAG_GFNI_OFF - disable use (and detection) of Galois Field extensions. + * IMB_FLAG_GFNI_OFF - disable use (and detection) of + * Galois Field extensions. * * @param reset_mgr if 0, IMB_MGR structure is not cleared, else it is. * @@ -186,11 +188,13 @@ IMB_MGR *imb_set_pointers_mb_mgr(void *mem_ptr, const uint64_t flags, unsigned i; /* Check if AESNI_EMU flag is set, needed to support AESNI emulation */ +#ifndef __aarch64__ #ifndef AESNI_EMU if (flags & IMB_FLAG_AESNI_OFF) { imb_set_errno(ptr, IMB_ERR_NO_AESNI_EMU); return NULL; } +#endif #endif if (reset_mgr) { /* Zero out MB_MGR memory */ @@ -200,6 +204,7 @@ IMB_MGR *imb_set_pointers_mb_mgr(void *mem_ptr, const uint64_t flags, /* Reset function pointers from previously used architecture */ switch (used_arch) { +#ifndef __aarch64__ case IMB_ARCH_NOAESNI: #ifdef AESNI_EMU init_mb_mgr_sse_no_aesni_internal(ptr, 0); @@ -221,6 +226,14 @@ IMB_MGR *imb_set_pointers_mb_mgr(void *mem_ptr, const uint64_t flags, case IMB_ARCH_AVX512: init_mb_mgr_avx512_internal(ptr, 0); break; +#else + case IMB_ARCH_NOAESNI: + init_mb_mgr_aarch64_no_aesni_internal(ptr, 0); + break; + case IMB_ARCH_AARCH64: + init_mb_mgr_aarch64_internal(ptr, 0); + break; +#endif default: break; } @@ -247,14 +260,11 @@ alloc_aligned_mem(const size_t size) void *ptr; #ifdef LINUX - if (posix_memalign((void **)&ptr, ALIGNMENT, size)) - return NULL; + ptr = memalign(ALIGNMENT, size); #else ptr = _aligned_malloc(size, ALIGNMENT); #endif - IMB_ASSERT(ptr != NULL); - return ptr; } @@ -278,7 +288,8 @@ free_mem(void *ptr) * IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, * currently SHANI is only available for SSE * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. - * IMB_FLAG_GFNI_OFF - disable use (and detection) of Galois Field extensions. + * IMB_FLAG_GFNI_OFF - disable use (and detection) of + * Galois Field extensions. * * @return Pointer to allocated memory for MB_MGR structure * @retval NULL on allocation error @@ -288,11 +299,13 @@ IMB_MGR *alloc_mb_mgr(uint64_t flags) IMB_MGR *ptr = NULL; /* Check if AESNI_EMU flag is set, needed to support AESNI emulation */ +#ifndef __aarch64__ #ifndef AESNI_EMU if (flags & IMB_FLAG_AESNI_OFF) { imb_set_errno(ptr, IMB_ERR_NO_AESNI_EMU); return NULL; } +#endif #endif ptr = alloc_aligned_mem(imb_get_mb_mgr_size()); IMB_ASSERT(ptr != NULL); diff --git a/lib/x86_64/atomic.asm b/lib/x86_64/atomic.asm new file mode 100644 index 0000000000000000000000000000000000000000..af6a191c71c6ce2c6975216fde448542f69dbf67 --- /dev/null +++ b/lib/x86_64/atomic.asm @@ -0,0 +1,61 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; function to wrap cpuid opcode across OS versions +%include "include/os.inc" +%include "include/reg_sizes.inc" + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%endif + +mksection .text + +;; +;; Post-increment atomic 64-bit increment +;; +;; Parameters: +;; [in] counter - pointer to a 64-bit counter +;; +;; uint64_t atomic_uint64_inc(uint64_t *counter) + +MKGLOBAL(atomic_uint64_inc,function,internal) +atomic_uint64_inc: + mov rax, [arg1] +atomic_uint64_loop: + lea r11, [rax + 1] + lock cmpxchg [arg1], r11 ;; compare counter against RAX, if not changed then store R11 in to counter + jnz atomic_uint64_loop ;; if counter changed between load and cmpxchg then load counter into RAX & try again + ret ;; return current counter value through RAX + +mksection stack-noexec diff --git a/lib/x86_64/chacha20_poly1305.c b/lib/x86_64/chacha20_poly1305.c index bad8c27859b2221b9aadb572243f3f1c9134cb90..ba69cd2e9f376f281f032ea336494e760353e95f 100644 --- a/lib/x86_64/chacha20_poly1305.c +++ b/lib/x86_64/chacha20_poly1305.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -75,6 +75,10 @@ void poly1305_aead_update(const void *msg, const uint64_t msg_len, key); else poly1305_aead_update_avx512(msg, msg_len, hash, key); +#ifdef AVX_IFMA + } else if (arch == IMB_ARCH_AVX2 && ifma) { + poly1305_aead_update_fma_avx2(msg, msg_len, hash, key); +#endif } else poly1305_aead_update_scalar(msg, msg_len, hash, key); } @@ -88,6 +92,10 @@ void poly1305_aead_complete(const void *hash, const void *key, poly1305_aead_complete_fma_avx512(hash, key, tag); else poly1305_aead_complete_avx512(hash, key, tag); +#ifdef AVX_IFMA + } else if (arch == IMB_ARCH_AVX2 && ifma) { + poly1305_aead_complete_fma_avx2(hash, key, tag); +#endif } else poly1305_aead_complete_scalar(hash, key, tag); } @@ -548,11 +556,13 @@ IMB_JOB *aead_chacha20_poly1305_sgl(IMB_JOB *job, const IMB_ARCH arch, break; case IMB_SGL_ALL: default: - init_chacha20_poly1305_direct(job->enc_keys, - ctx, - job->iv, job->u.CHACHA20_POLY1305.aad, - job->u.CHACHA20_POLY1305.aad_len_in_bytes, - arch, 0, ifma); + init_chacha20_poly1305_direct( + job->enc_keys, + ctx, + job->iv, + job->u.CHACHA20_POLY1305.aad, + job->u.CHACHA20_POLY1305.aad_len_in_bytes, + arch, 0, ifma); for (i = 0; i < job->num_sgl_io_segs; i++) update_chacha20_poly1305_direct(job->enc_keys, ctx, @@ -564,9 +574,9 @@ IMB_JOB *aead_chacha20_poly1305_sgl(IMB_JOB *job, const IMB_ARCH arch, 0, ifma); finalize_chacha20_poly1305_direct(ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes, - arch, 0, ifma); + job->auth_tag_output, + job->auth_tag_output_len_in_bytes, + arch, 0, ifma); /* Clear sensitive data from the context */ #ifdef SAFE_DATA clear_mem(ctx->last_ks, sizeof(ctx->last_ks)); @@ -699,8 +709,10 @@ IMB_JOB *aead_chacha20_poly1305_avx(IMB_MGR *mgr, IMB_JOB *job) IMB_DLL_LOCAL IMB_JOB *aead_chacha20_poly1305_avx2(IMB_MGR *mgr, IMB_JOB *job) { - (void) mgr; - return aead_chacha20_poly1305(job, IMB_ARCH_AVX2, 0); + if (mgr->features & IMB_FEATURE_AVX_IFMA) + return aead_chacha20_poly1305(job, IMB_ARCH_AVX2, 1); + else + return aead_chacha20_poly1305(job, IMB_ARCH_AVX2, 0); } IMB_DLL_LOCAL @@ -729,8 +741,10 @@ IMB_JOB *aead_chacha20_poly1305_sgl_avx(IMB_MGR *mgr, IMB_JOB *job) IMB_DLL_LOCAL IMB_JOB *aead_chacha20_poly1305_sgl_avx2(IMB_MGR *mgr, IMB_JOB *job) { - (void) mgr; - return aead_chacha20_poly1305_sgl(job, IMB_ARCH_AVX2, 0); + if (mgr->features & IMB_FEATURE_AVX_IFMA) + return aead_chacha20_poly1305_sgl(job, IMB_ARCH_AVX2, 1); + else + return aead_chacha20_poly1305_sgl(job, IMB_ARCH_AVX2, 0); } IMB_DLL_LOCAL @@ -762,6 +776,28 @@ void init_chacha20_poly1305_avx(const void *key, aad_len, IMB_ARCH_AVX, 1, 0); } +IMB_DLL_LOCAL +void init_chacha20_poly1305_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + const void *iv, const void *aad, + const uint64_t aad_len) +{ + init_chacha20_poly1305_direct(key, ctx, iv, aad, + aad_len, IMB_ARCH_AVX2, 1, 0); +} + +#ifdef AVX_IFMA +IMB_DLL_LOCAL +void init_chacha20_poly1305_fma_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + const void *iv, const void *aad, + const uint64_t aad_len) +{ + init_chacha20_poly1305_direct(key, ctx, iv, aad, + aad_len, IMB_ARCH_AVX2, 1, 1); +} +#endif + IMB_DLL_LOCAL void init_chacha20_poly1305_avx512(const void *key, struct chacha20_poly1305_context_data *ctx, @@ -809,6 +845,16 @@ void update_enc_chacha20_poly1305_avx2(const void *key, IMB_DIR_ENCRYPT, IMB_ARCH_AVX2, 1, 0); } +#ifdef AVX_IFMA +void update_enc_chacha20_poly1305_fma_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + void *dst, const void *src, + const uint64_t len) +{ + update_chacha20_poly1305_direct(key, ctx, dst, src, len, + IMB_DIR_ENCRYPT, IMB_ARCH_AVX2, 1, 1); +} +#endif void update_enc_chacha20_poly1305_avx512(const void *key, struct chacha20_poly1305_context_data *ctx, @@ -855,6 +901,17 @@ void update_dec_chacha20_poly1305_avx2(const void *key, IMB_DIR_DECRYPT, IMB_ARCH_AVX2, 1, 0); } +#ifdef AVX_IFMA +void update_dec_chacha20_poly1305_fma_avx2(const void *key, + struct chacha20_poly1305_context_data *ctx, + void *dst, const void *src, + const uint64_t len) +{ + update_chacha20_poly1305_direct(key, ctx, dst, src, len, + IMB_DIR_DECRYPT, IMB_ARCH_AVX2, 1, 1); +} +#endif + void update_dec_chacha20_poly1305_avx512(const void *key, struct chacha20_poly1305_context_data *ctx, void *dst, const void *src, @@ -876,14 +933,33 @@ void update_dec_chacha20_poly1305_fma_avx512(const void *key, void finalize_chacha20_poly1305_sse(struct chacha20_poly1305_context_data *ctx, void *tag, const uint64_t tag_len) { - finalize_chacha20_poly1305_direct(ctx, tag, tag_len, IMB_ARCH_SSE, 1, 0); + finalize_chacha20_poly1305_direct(ctx, tag, tag_len, + IMB_ARCH_SSE, 1, 0); } void finalize_chacha20_poly1305_avx(struct chacha20_poly1305_context_data *ctx, void *tag, const uint64_t tag_len) { - finalize_chacha20_poly1305_direct(ctx, tag, tag_len, IMB_ARCH_AVX, 1, 0); + finalize_chacha20_poly1305_direct(ctx, tag, tag_len, + IMB_ARCH_AVX, 1, 0); +} + +void finalize_chacha20_poly1305_avx2(struct chacha20_poly1305_context_data *ctx, + void *tag, const uint64_t tag_len) +{ + finalize_chacha20_poly1305_direct(ctx, tag, tag_len, + IMB_ARCH_AVX2, 1, 0); +} + +#ifdef AVX_IFMA +void finalize_chacha20_poly1305_fma_avx2(struct + chacha20_poly1305_context_data *ctx, + void *tag, const uint64_t tag_len) +{ + finalize_chacha20_poly1305_direct(ctx, tag, tag_len, + IMB_ARCH_AVX2, 1, 1); } +#endif void finalize_chacha20_poly1305_avx512( struct chacha20_poly1305_context_data *ctx, diff --git a/lib/x86_64/cipher_suite_id.c b/lib/x86_64/cipher_suite_id.c new file mode 100644 index 0000000000000000000000000000000000000000..a8142be0243446594fabdee2f9faa2d2024f50dd --- /dev/null +++ b/lib/x86_64/cipher_suite_id.c @@ -0,0 +1,83 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "ipsec-mb.h" +#include "include/error.h" +#ifndef __aarch64__ +#include "include/arch_x86_64.h" +#else +#include +#endif + +IMB_DLL_EXPORT uint32_t imb_set_session(IMB_MGR *state, IMB_JOB *job) +{ + struct { + uint16_t key_len; + uint16_t hash_alg; + uint16_t cipher_mode; + uint16_t cipher_dir; + uint64_t counter; + } extract; + static uint64_t counter = 1; + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return 0; + } + if (job == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } + imb_set_errno(state, 0); +#endif + /* Fill in suite_id[] structure in \a job */ + state->set_suite_id(state, job); + + /** + * Calculate and set session_id in \a job + * Set up extract structure: + * - collect session specific data + * - plus counter value (secures different ID for the same + * cipher suite params) + * Session ID is CRC calculated on the extract structure. + */ + extract.key_len = (uint16_t) job->key_len_in_bytes; + extract.hash_alg = (uint16_t) job->hash_alg; + extract.cipher_mode = (uint16_t) job->cipher_mode ; + extract.cipher_dir = (uint16_t) job->cipher_direction; +#ifndef __aarch64__ + extract.counter = atomic_uint64_inc(&counter); +#else + extract.counter = atomic_fetch_add(&counter, 1); +#endif + const uint32_t id = + IMB_CRC32_WIMAX_OFDMA_DATA(state, &extract, sizeof(extract)); + + job->session_id = id; + return id; +} diff --git a/lib/x86_64/clear_regs_mem_fns.asm b/lib/x86_64/clear_regs_mem_fns.asm index a58c07bf79081b75afad01ab083937d87dc30837..0238b99539fe28d606002b8accbff193939fd3d1 100644 --- a/lib/x86_64/clear_regs_mem_fns.asm +++ b/lib/x86_64/clear_regs_mem_fns.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,8 +25,8 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/clear_regs.inc" %include "include/cet.inc" mksection .text ; diff --git a/lib/x86_64/const.asm b/lib/x86_64/const.asm index 42c3ffba1d4b09a70dae16a6619f8631c3a82798..9ac0484bb4f6b6d6e34f699d4fab0860ffd1d9cd 100644 --- a/lib/x86_64/const.asm +++ b/lib/x86_64/const.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2018-2022, Intel Corporation +;; Copyright (c) 2018-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" mksection .rodata default rel diff --git a/lib/x86_64/constant_lookup_fns.asm b/lib/x86_64/constant_lookup_fns.asm index 2e40f1bbcfeaab059525550fc3d694ca40a11b74..fbb2d0e618f56fc991919498dc3548a085b0ee0a 100644 --- a/lib/x86_64/constant_lookup_fns.asm +++ b/lib/x86_64/constant_lookup_fns.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/constant_lookup.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/constant_lookup.inc" mksection .rodata default rel diff --git a/lib/x86_64/cpu_feature.c b/lib/x86_64/cpu_feature.c index 5248853bc8eadecd280c4aaa8be200f6aafd5ce3..2c0555dd31156ed30bdf41b0224438b3f21503c1 100644 --- a/lib/x86_64/cpu_feature.c +++ b/lib/x86_64/cpu_feature.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -26,9 +26,6 @@ *******************************************************************************/ #include -#ifdef __WIN32 -#include -#endif #include "cpu_feature.h" @@ -41,6 +38,7 @@ struct cpuid_regs { static struct cpuid_regs cpuid_1_0; static struct cpuid_regs cpuid_7_0; +static struct cpuid_regs cpuid_7_1; /* * A C wrapper for CPUID opcode @@ -50,33 +48,9 @@ static struct cpuid_regs cpuid_7_0; * [in] subleaf - CPUID sub-leaf number (ECX) * [out] out - registers structure to store results of CPUID into */ -static void -__mbcpuid(const unsigned leaf, const unsigned subleaf, struct cpuid_regs *out) -{ -#ifdef _WIN32 - /* Windows */ - int regs[4]; - - __cpuidex(regs, leaf, subleaf); - out->eax = regs[0]; - out->ebx = regs[1]; - out->ecx = regs[2]; - out->edx = regs[3]; -#else - /* Linux */ - asm volatile("mov %4, %%eax\n\t" - "mov %5, %%ecx\n\t" - "cpuid\n\t" - "mov %%eax, %0\n\t" - "mov %%ebx, %1\n\t" - "mov %%ecx, %2\n\t" - "mov %%edx, %3\n\t" - : "=g" (out->eax), "=g" (out->ebx), "=g" (out->ecx), - "=g" (out->edx) - : "g" (leaf), "g" (subleaf) - : "%eax", "%ebx", "%ecx", "%edx"); -#endif /* Linux */ -} +void mbcpuid(const unsigned leaf, + const unsigned subleaf, + struct cpuid_regs *out); static uint32_t detect_shani(void) { @@ -174,6 +148,16 @@ static uint32_t detect_avx512_ifma(void) return (cpuid_7_0.ebx & (1UL << 21)); } +static uint32_t detect_avx_ifma(void) +{ + /* Check presence of AVX-IFMA - bit 23 of EAX */ +#ifdef AVX_IFMA + return (cpuid_7_1.eax & (1UL << 23)); +#else + return 0; +#endif +} + static uint32_t detect_bmi2(void) { /* Check presence of BMI2 - bit 8 of EBX */ @@ -204,6 +188,7 @@ uint64_t cpu_feature_detect(void) { 7, IMB_FEATURE_GFNI, detect_gfni }, { 7, IMB_FEATURE_AVX512_IFMA, detect_avx512_ifma }, { 7, IMB_FEATURE_BMI2, detect_bmi2 }, + { 7, IMB_FEATURE_AVX_IFMA, detect_avx_ifma }, }; struct cpuid_regs r; unsigned hi_leaf_number = 0; @@ -211,15 +196,18 @@ uint64_t cpu_feature_detect(void) unsigned i; /* Get highest supported CPUID leaf number */ - __mbcpuid(0x0, 0x0, &r); + mbcpuid(0x0, 0x0, &r); hi_leaf_number = r.eax; /* Get the most common CPUID leafs to speed up the detection */ if (hi_leaf_number >= 1) - __mbcpuid(0x1, 0x0, &cpuid_1_0); + mbcpuid(0x1, 0x0, &cpuid_1_0); + + if (hi_leaf_number >= 7) + mbcpuid(0x7, 0x0, &cpuid_7_0); if (hi_leaf_number >= 7) - __mbcpuid(0x7, 0x0, &cpuid_7_0); + mbcpuid(0x7, 0x1, &cpuid_7_1); for (i = 0; i < IMB_DIM(feat_tab); i++) { if (hi_leaf_number < feat_tab[i].req_leaf_number) diff --git a/lib/x86_64/crc32_const.asm b/lib/x86_64/crc32_const.asm index f94619d766ab73c4596adc6181dd1940e5e008da..d0cea5dd4228f92d072b6473b077920f5b40b0ed 100644 --- a/lib/x86_64/crc32_const.asm +++ b/lib/x86_64/crc32_const.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" [bits 64] default rel diff --git a/lib/x86_64/crc32_refl_const.asm b/lib/x86_64/crc32_refl_const.asm index 2b99a0266784cc591faea0544a3f6a1b8175e55b..ae3c33fea38879df6b3ea2e50fa135cf8f562501 100644 --- a/lib/x86_64/crc32_refl_const.asm +++ b/lib/x86_64/crc32_refl_const.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" [bits 64] default rel diff --git a/lib/x86_64/des_basic.c b/lib/x86_64/des_basic.c index e28c9ce4ea1ef7ad607084c2e6a89e4c07b268d0..2a08ccea128820051f49a55e67a57a4a1d852a5a 100644 --- a/lib/x86_64/des_basic.c +++ b/lib/x86_64/des_basic.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -751,3 +751,11 @@ docsis_des_dec_basic(const void *input, void *output, const int size, iv = in_block; } } + +IMB_DLL_EXPORT +void +des_cfb_one(void *output, const void *input, const uint64_t *iv, + const uint64_t *ks, const int size) +{ + cfb_one_basic(input, output, size, ks, iv); +} diff --git a/lib/x86_64/des_key.c b/lib/x86_64/des_key.c index 0e110a614f0a0304ddc6993dce67b98bb28c4a62..a80f305511730cf01ac0ca07919f02ebc2458525 100644 --- a/lib/x86_64/des_key.c +++ b/lib/x86_64/des_key.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/x86_64/error.c b/lib/x86_64/error.c index 61dd46d94d6374fe36d416aefb4a2851a16f66b1..dbcbef7050c093bfb78ea0494dcd53260813bc6b 100644 --- a/lib/x86_64/error.c +++ b/lib/x86_64/error.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -39,7 +39,7 @@ #pragma warning(disable : 4996) #endif -IMB_DLL_LOCAL int imb_errno = 0; +IMB_DLL_LOCAL volatile int imb_errno; IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_NULL_MBMGR, IMB_ERR_JOB_NULL_SRC, @@ -92,16 +92,16 @@ IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_NULL_BURST, IMB_ERR_BURST_SIZE, IMB_ERR_BURST_OOO, - IMB_ERR_SELFTEST + IMB_ERR_SELFTEST, + IMB_ERR_BURST_SUITE_ID }; -#ifdef DEBUG -static_assert((IMB_DIM(imb_errno_types) + 1) == (IMB_ERR_MAX - IMB_ERR_MIN), - "imb_errno_types[] mismatch vs enum IMB_ERR"); -#endif - int imb_get_errno(IMB_MGR *mb_mgr) { + /* check for imb_errno_types[] mismatch vs enum IMB_ERR */ + IMB_ASSERT((IMB_DIM(imb_errno_types) + 1) == + (IMB_ERR_MAX - IMB_ERR_MIN)); + /* try get IMB_MGR error status first */ if (mb_mgr != NULL && mb_mgr->imb_errno) return mb_mgr->imb_errno; @@ -226,6 +226,8 @@ imb_get_strerror(int errnum) return "Burst jobs out of order"; case IMB_ERR_SELFTEST: return "Self-test failed"; + case IMB_ERR_BURST_SUITE_ID: + return "Invalid cipher suite ID (async burst API)"; default: return strerror(errnum); } diff --git a/lib/x86_64/gcm.c b/lib/x86_64/gcm.c index 50963aace25549dea050a895f190fde106390379..e23ac03b03501fb0cc8f88b22874b11fbc2a49cc 100644 --- a/lib/x86_64/gcm.c +++ b/lib/x86_64/gcm.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -85,21 +85,7 @@ void aes_gcm_pre_128_sse_no_aesni(const void *key, void aes_gcm_pre_128_avx_gen2(const void *key, struct gcm_key_data *key_data) { -#ifdef SAFE_PARAM - /* reset error status */ - imb_set_errno(NULL, 0); - - if (key == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_KEY); - return; - } - if (key_data == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_EXP_KEY); - return; - } -#endif - aes_keyexp_128_enc_avx(key, key_data->expanded_keys); - aes_gcm_precomp_128_avx_gen2(key_data); + aes_gcm_pre_128_sse(key, key_data); } void aes_gcm_pre_128_avx_gen4(const void *key, struct gcm_key_data *key_data) @@ -202,21 +188,7 @@ void aes_gcm_pre_192_sse_no_aesni(const void *key, void aes_gcm_pre_192_avx_gen2(const void *key, struct gcm_key_data *key_data) { -#ifdef SAFE_PARAM - /* reset error status */ - imb_set_errno(NULL, 0); - - if (key == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_KEY); - return; - } - if (key_data == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_EXP_KEY); - return; - } -#endif - aes_keyexp_192_enc_avx(key, key_data->expanded_keys); - aes_gcm_precomp_192_avx_gen2(key_data); + aes_gcm_pre_192_sse(key, key_data); } void aes_gcm_pre_192_avx_gen4(const void *key, struct gcm_key_data *key_data) @@ -319,21 +291,7 @@ void aes_gcm_pre_256_sse_no_aesni(const void *key, void aes_gcm_pre_256_avx_gen2(const void *key, struct gcm_key_data *key_data) { -#ifdef SAFE_PARAM - /* reset error status */ - imb_set_errno(NULL, 0); - - if (key == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_KEY); - return; - } - if (key_data == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_EXP_KEY); - return; - } -#endif - aes_keyexp_256_enc_avx(key, key_data->expanded_keys); - aes_gcm_precomp_256_avx_gen2(key_data); + aes_gcm_pre_256_sse(key, key_data); } void aes_gcm_pre_256_avx_gen4(const void *key, struct gcm_key_data *key_data) @@ -459,8 +417,8 @@ imb_aes_gmac_init_128_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, const uint64_t iv_len) { - aes_gcm_init_var_iv_128_avx_gen2(key_data, context_data, iv, - iv_len, NULL, 0); + aes_gcm_init_var_iv_128_sse(key_data, context_data, iv, + iv_len, NULL, 0); } void @@ -469,8 +427,8 @@ imb_aes_gmac_init_192_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, const uint64_t iv_len) { - aes_gcm_init_var_iv_192_avx_gen2(key_data, context_data, iv, - iv_len, NULL, 0); + aes_gcm_init_var_iv_192_sse(key_data, context_data, iv, + iv_len, NULL, 0); } void @@ -479,8 +437,8 @@ imb_aes_gmac_init_256_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, const uint64_t iv_len) { - aes_gcm_init_var_iv_256_avx_gen2(key_data, context_data, iv, - iv_len, NULL, 0); + aes_gcm_init_var_iv_256_sse(key_data, context_data, iv, + iv_len, NULL, 0); } void @@ -489,8 +447,8 @@ imb_aes_gmac_finalize_128_avx_gen2(const struct gcm_key_data *key_data, uint8_t *auth_tag, const uint64_t auth_tag_len) { - aes_gcm_enc_128_finalize_avx_gen2(key_data, context_data, auth_tag, - auth_tag_len); + aes_gcm_enc_128_finalize_sse(key_data, context_data, auth_tag, + auth_tag_len); } void @@ -499,8 +457,8 @@ imb_aes_gmac_finalize_192_avx_gen2(const struct gcm_key_data *key_data, uint8_t *auth_tag, const uint64_t auth_tag_len) { - aes_gcm_enc_192_finalize_avx_gen2(key_data, context_data, auth_tag, - auth_tag_len); + aes_gcm_enc_192_finalize_sse(key_data, context_data, auth_tag, + auth_tag_len); } void @@ -509,8 +467,8 @@ imb_aes_gmac_finalize_256_avx_gen2(const struct gcm_key_data *key_data, uint8_t *auth_tag, const uint64_t auth_tag_len) { - aes_gcm_enc_256_finalize_avx_gen2(key_data, context_data, auth_tag, - auth_tag_len); + aes_gcm_enc_256_finalize_sse(key_data, context_data, auth_tag, + auth_tag_len); } #ifdef AESNI_EMU diff --git a/lib/x86_64/hmac_ipad_opad.c b/lib/x86_64/hmac_ipad_opad.c new file mode 100644 index 0000000000000000000000000000000000000000..56f254a24a3cdab70a530f916b3ce442d539b588 --- /dev/null +++ b/lib/x86_64/hmac_ipad_opad.c @@ -0,0 +1,173 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "include/error.h" + +IMB_DLL_EXPORT +void +imb_hmac_ipad_opad(IMB_MGR *mb_mgr, const IMB_HASH_ALG sha_type, + const void *pkey, const size_t key_len, + void *ipad_hash, void *opad_hash) +{ +#ifdef SAFE_PARAM + if (mb_mgr == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return; + } + if (pkey == NULL) { + imb_set_errno(mb_mgr, IMB_ERR_NULL_KEY); + return; + } + imb_set_errno(mb_mgr, 0); +#endif + uint32_t i = 0; + size_t local_key_len = 0; + + switch (sha_type) { + case IMB_AUTH_HMAC_SHA_1: + local_key_len = (key_len <= IMB_SHA1_BLOCK_SIZE) ? + key_len : IMB_SHA1_DIGEST_SIZE_IN_BYTES; + break; + case IMB_AUTH_HMAC_SHA_224: + local_key_len = (key_len <= IMB_SHA_224_BLOCK_SIZE) ? + key_len : IMB_SHA224_DIGEST_SIZE_IN_BYTES; + break; + case IMB_AUTH_HMAC_SHA_256: + local_key_len = (key_len <= IMB_SHA_256_BLOCK_SIZE) ? + key_len : IMB_SHA256_DIGEST_SIZE_IN_BYTES; + break; + case IMB_AUTH_HMAC_SHA_384: + local_key_len = (key_len <= IMB_SHA_384_BLOCK_SIZE) ? + key_len : IMB_SHA384_DIGEST_SIZE_IN_BYTES; + break; + case IMB_AUTH_HMAC_SHA_512: + local_key_len = (key_len <= IMB_SHA_512_BLOCK_SIZE) ? + key_len : IMB_SHA512_DIGEST_SIZE_IN_BYTES; + break; + case IMB_AUTH_MD5: + if (key_len <= IMB_MD5_BLOCK_SIZE) + local_key_len = key_len; + else { + /** + * Key lengths longer than MD5 block + * size not supported + */ + imb_set_errno(NULL, IMB_ERR_KEY_LEN); + return; + } + break; + default: + imb_set_errno(NULL, IMB_ERR_HASH_ALGO); + return; + } + uint8_t key[IMB_SHA_512_BLOCK_SIZE]; + uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; + + /* prepare the key */ + if (local_key_len == key_len) { + memcpy(key, pkey, key_len); + } else + switch (sha_type) { + case IMB_AUTH_HMAC_SHA_1: + IMB_SHA1(mb_mgr, pkey, key_len, key); + break; + case IMB_AUTH_HMAC_SHA_224: + IMB_SHA224(mb_mgr, pkey, key_len, key); + break; + case IMB_AUTH_HMAC_SHA_256: + IMB_SHA256(mb_mgr, pkey, key_len, key); + break; + case IMB_AUTH_HMAC_SHA_384: + IMB_SHA384(mb_mgr, pkey, key_len, key); + break; + default: /* For SHA-512 */ + IMB_SHA512(mb_mgr, pkey, key_len, key); + } + + /* compute ipad hash */ + if (ipad_hash != NULL) { + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < local_key_len; i++) + buf[i] ^= key[i]; + switch (sha_type) { + case IMB_AUTH_HMAC_SHA_1: + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case IMB_AUTH_HMAC_SHA_224: + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case IMB_AUTH_HMAC_SHA_256: + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case IMB_AUTH_HMAC_SHA_384: + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case IMB_AUTH_HMAC_SHA_512: + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + default: /* For MD5*/ + IMB_MD5_ONE_BLOCK(mb_mgr, buf, ipad_hash); + } + } + + /* compute opad hash */ + if (opad_hash != NULL) { + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < local_key_len; i++) + buf[i] ^= key[i]; + switch (sha_type) { + case IMB_AUTH_HMAC_SHA_1: + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case IMB_AUTH_HMAC_SHA_224: + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case IMB_AUTH_HMAC_SHA_256: + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case IMB_AUTH_HMAC_SHA_384: + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case IMB_AUTH_HMAC_SHA_512: + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + default: /* For MD5 */ + IMB_MD5_ONE_BLOCK(mb_mgr, buf, opad_hash); + } + } + +#ifdef SAFE_DATA + imb_clear_mem(key, sizeof(key)); + imb_clear_mem(buf, sizeof(buf)); +#endif +} diff --git a/lib/x86_64/kasumi_iv.c b/lib/x86_64/kasumi_iv.c index 3a54c0cd2178bd6471df5e44ae0432efcbd37e88..6db884b2c499699fdc48e3ed8d002cab383c1a80 100644 --- a/lib/x86_64/kasumi_iv.c +++ b/lib/x86_64/kasumi_iv.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/x86_64/mb_mgr_auto.c b/lib/x86_64/mb_mgr_auto.c index ccf4a18acc3fe467adf41aac2e7fe74112af7898..beac5599387e157ce6e17a31168a9b2128dfe06c 100644 --- a/lib/x86_64/mb_mgr_auto.c +++ b/lib/x86_64/mb_mgr_auto.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -74,7 +74,8 @@ init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch) } #ifdef AESNI_EMU - if ((state->features & IMB_CPUFLAGS_NO_AESNI) == IMB_CPUFLAGS_NO_AESNI) { + if ((state->features & IMB_CPUFLAGS_NO_AESNI) + == IMB_CPUFLAGS_NO_AESNI) { init_mb_mgr_sse_no_aesni(state); arch_detected = IMB_ARCH_NOAESNI; goto init_mb_mgr_auto_ret; diff --git a/lib/x86_64/mbcpuid.asm b/lib/x86_64/mbcpuid.asm new file mode 100644 index 0000000000000000000000000000000000000000..fc5c873d9907a2ce54d657c73d7fd92ae1be9da6 --- /dev/null +++ b/lib/x86_64/mbcpuid.asm @@ -0,0 +1,72 @@ +;; +;; Copyright (c) 2023, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; function to wrap cpuid opcode across OS versions +%include "include/os.inc" +%include "include/reg_sizes.inc" + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%endif + +mksection .text + +;; +;; Wrapper for CPUID opcode +;; +;; Parameters: +;; [in] leaf - CPUID leaf number (EAX) +;; [in] subleaf - CPUID sub-leaf number (ECX) +;; [out] out - registers structure to store results of CPUID into +;; +;; void mbcpuid(const unsigned leaf, const unsigned subleaf, struct cpuid_regs *out) + +MKGLOBAL(mbcpuid,function,internal) +mbcpuid: + push rbx + + mov r11, arg3 ;; arg3 will get overwritten with cpuid on sysv + mov eax, DWORD(arg1) + mov ecx, DWORD(arg2) + + cpuid + + mov [r11 + 0*4], eax + mov [r11 + 1*4], ebx + mov [r11 + 2*4], ecx + mov [r11 + 3*4], edx + + pop rbx + ret + +mksection stack-noexec diff --git a/lib/x86_64/md5_one_block.c b/lib/x86_64/md5_one_block.c index bf4f6668e09501a8246f1ea2a45c3dcd1b2315ca..9de294e1b8648c1e5d727e61456cc47ba481a9f3 100644 --- a/lib/x86_64/md5_one_block.c +++ b/lib/x86_64/md5_one_block.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 5224ea920e71510446c931a79555fdd1a02cfc7f..a582d7781d3de63b32c5055a5346d67e679bd6a0 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -32,6 +32,7 @@ #include "include/ooo_mgr_reset.h" #include /* offsetof() */ +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) { @@ -104,7 +105,7 @@ void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_AES_XCBC_OOO *p_mgr = (MB_MGR_AES_XCBC_OOO *) p_ooo_mgr; unsigned i; - + memset(p_mgr, 0, offsetof(MB_MGR_AES_XCBC_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -124,7 +125,7 @@ void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_HMAC_SHA_1_OOO *p_mgr = (MB_MGR_HMAC_SHA_1_OOO *) p_ooo_mgr; unsigned i; - + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_1_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -154,7 +155,7 @@ void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_HMAC_SHA_256_OOO *p_mgr = (MB_MGR_HMAC_SHA_256_OOO *) p_ooo_mgr; unsigned i; - + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_256_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -184,7 +185,7 @@ void ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_HMAC_SHA_256_OOO *p_mgr = (MB_MGR_HMAC_SHA_256_OOO *) p_ooo_mgr; unsigned i; - + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_256_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -214,7 +215,7 @@ void ooo_mgr_hmac_sha384_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_HMAC_SHA_512_OOO *p_mgr = (MB_MGR_HMAC_SHA_512_OOO *) p_ooo_mgr; unsigned i; - + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_512_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -250,7 +251,7 @@ void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_HMAC_SHA_512_OOO *p_mgr = (MB_MGR_HMAC_SHA_512_OOO *) p_ooo_mgr; unsigned i; - + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_512_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -286,7 +287,7 @@ void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_HMAC_MD5_OOO *p_mgr = (MB_MGR_HMAC_MD5_OOO *) p_ooo_mgr; unsigned i; - + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_MD5_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -305,12 +306,13 @@ void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == AVX2_NUM_MD5_LANES) p_mgr->unused_lanes = 0xFEDCBA9876543210; } +#endif /* __aarch64__ */ IMB_DLL_LOCAL void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_ZUC_OOO *p_mgr = (MB_MGR_ZUC_OOO *) p_ooo_mgr; - + memset(p_mgr, 0, offsetof(MB_MGR_ZUC_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); @@ -326,11 +328,12 @@ void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes) } } +#ifndef __aarch64__ IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_SHA_1_OOO *p_mgr = (MB_MGR_SHA_1_OOO *) p_ooo_mgr; - + memset(p_mgr, 0, offsetof(MB_MGR_SHA_1_OOO,road_block)); if (num_lanes == 2) @@ -379,12 +382,13 @@ IMB_DLL_LOCAL void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_DES_OOO *p_mgr = (MB_MGR_DES_OOO *) p_ooo_mgr; - + memset(p_mgr, 0, offsetof(MB_MGR_DES_OOO,road_block)); if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; } +#endif /* __aarch64__ */ IMB_DLL_LOCAL void ooo_mgr_snow3g_reset(void *p_ooo_mgr, const unsigned num_lanes) diff --git a/lib/x86_64/poly1305.asm b/lib/x86_64/poly1305.asm index 46051ee3d90db82c06fb7ad462ae6ac3cfa69a37..75e7143e3b00b061d7e1f1f4b47623924cfe5137 100644 --- a/lib/x86_64/poly1305.asm +++ b/lib/x86_64/poly1305.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2020-2022, Intel Corporation +;; Copyright (c) 2020-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -33,11 +33,11 @@ ;; "The design of Poly1305" ;; http://loup-vaillant.fr/tutorials/poly1305-design -%include "include/os.asm" -%include "include/reg_sizes.asm" -%include "include/memcpy.asm" -%include "include/imb_job.asm" -%include "include/clear_regs.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" +%include "include/memcpy.inc" +%include "include/imb_job.inc" +%include "include/clear_regs.inc" [bits 64] default rel diff --git a/lib/x86_64/quic_aes_gcm.c b/lib/x86_64/quic_aes_gcm.c new file mode 100644 index 0000000000000000000000000000000000000000..50c5f2af5464b5fcb33fde4dad4dc2fc425d4b69 --- /dev/null +++ b/lib/x86_64/quic_aes_gcm.c @@ -0,0 +1,179 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "ipsec-mb.h" +#include "include/error.h" + +IMB_DLL_EXPORT void +imb_quic_aes_gcm(IMB_MGR *state, + const struct gcm_key_data *key_data, + const IMB_KEY_SIZE_BYTES key_size, + const IMB_CIPHER_DIRECTION cipher_dir, + void *dst_ptr_array[], + const void * const src_ptr_array[], + const uint64_t len_array[], + const void * const iv_ptr_array[], + const void * const aad_ptr_array[], + const uint64_t aad_len, + void *tag_ptr_array[], + const uint64_t tag_len, + const uint64_t num_packets) +{ + /** + * @note 12 byte IV is assumed + * @note it can be out of place operation + * but AAD needs to be copied by the caller + */ + struct gcm_context_data ctx; + uint64_t n; + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return; + } + if (key_data == NULL) { + imb_set_errno(state, IMB_ERR_NULL_EXP_KEY); + return; + } + if (dst_ptr_array == NULL) { + imb_set_errno(state, IMB_ERR_NULL_DST); + return; + } + if (src_ptr_array == NULL) { + imb_set_errno(state, IMB_ERR_NULL_SRC); + return; + } + if (iv_ptr_array == NULL) { + imb_set_errno(state, IMB_ERR_NULL_IV); + return; + } + if (aad_ptr_array == NULL) { + imb_set_errno(state, IMB_ERR_NULL_AAD); + return; + } + if (tag_ptr_array == NULL) { + imb_set_errno(state, IMB_ERR_NULL_AUTH); + return; + } + for (n = 0; n < num_packets; n++) { + if (dst_ptr_array[n] == NULL && len_array[n] != 0) { + imb_set_errno(state, IMB_ERR_NULL_DST); + return; + } + if (src_ptr_array[n] == NULL && len_array[n] != 0) { + imb_set_errno(state, IMB_ERR_NULL_SRC); + return; + } + if (iv_ptr_array[n] == NULL) { + imb_set_errno(state, IMB_ERR_NULL_IV); + return; + } + if (aad_ptr_array[n] == NULL && aad_len != 0) { + imb_set_errno(state, IMB_ERR_NULL_AAD); + return; + } + if (tag_ptr_array[n] == NULL) { + imb_set_errno(state, IMB_ERR_NULL_AUTH); + return; + } + } + switch (key_size) { + case IMB_KEY_128_BYTES: + case IMB_KEY_256_BYTES: + break; + case IMB_KEY_192_BYTES: + /* AES-192 is not supported by QUIC */ + default: + imb_set_errno(state, IMB_ERR_KEY_LEN); + return; + } + switch (cipher_dir) { + case IMB_DIR_ENCRYPT: + case IMB_DIR_DECRYPT: + break; + default: + imb_set_errno(state, IMB_ERR_JOB_CIPH_DIR); + return; + } +#endif /* SAFE_PARAM */ + + if (cipher_dir == IMB_DIR_ENCRYPT) { + if (key_size == IMB_KEY_128_BYTES) { + for (n = 0; n < num_packets; n++) { + IMB_AES128_GCM_ENC(state, key_data, &ctx, + dst_ptr_array[n], + src_ptr_array[n], + len_array[n], + iv_ptr_array[n], + aad_ptr_array[n], + aad_len, + tag_ptr_array[n], + tag_len); + } + } else /* assume 256-bits key */ { + for (n = 0; n < num_packets; n++) { + IMB_AES256_GCM_ENC(state, key_data, &ctx, + dst_ptr_array[n], + src_ptr_array[n], + len_array[n], + iv_ptr_array[n], + aad_ptr_array[n], + aad_len, + tag_ptr_array[n], + tag_len); + } + } + } else /* decrypt direction */ { + if (key_size == IMB_KEY_128_BYTES) { + for (n = 0; n < num_packets; n++) { + IMB_AES128_GCM_DEC(state, key_data, &ctx, + dst_ptr_array[n], + src_ptr_array[n], + len_array[n], + iv_ptr_array[n], + aad_ptr_array[n], + aad_len, + tag_ptr_array[n], + tag_len); + } + } else /* assume 256-bits key */ { + for (n = 0; n < num_packets; n++) { + IMB_AES256_GCM_DEC(state, key_data, &ctx, + dst_ptr_array[n], + src_ptr_array[n], + len_array[n], + iv_ptr_array[n], + aad_ptr_array[n], + aad_len, + tag_ptr_array[n], + tag_len); + } + } + } +} + diff --git a/lib/x86_64/quic_hp_aes_ecb.c b/lib/x86_64/quic_hp_aes_ecb.c new file mode 100644 index 0000000000000000000000000000000000000000..b2e7150cb684f894c036a34f798c3d0f720d2290 --- /dev/null +++ b/lib/x86_64/quic_hp_aes_ecb.c @@ -0,0 +1,87 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "ipsec-mb.h" +#include "include/error.h" + +IMB_DLL_EXPORT void +imb_quic_hp_aes_ecb(IMB_MGR *state, + const void *exp_key_data, + void *dst_ptr_array[], + const void * const src_ptr_array[], + const uint64_t num_packets, + const IMB_KEY_SIZE_BYTES key_size) +{ +#ifdef SAFE_PARAM + uint64_t i; + + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return; + } + if (exp_key_data == NULL) { + imb_set_errno(state, IMB_ERR_NULL_EXP_KEY); + return; + } + if (dst_ptr_array == NULL) { + imb_set_errno(state, IMB_ERR_NULL_DST); + return; + } + if (src_ptr_array == NULL) { + imb_set_errno(state, IMB_ERR_NULL_SRC); + return; + } + for (i = 0; i < num_packets; i++) { + if (dst_ptr_array[i] == NULL) { + imb_set_errno(state, IMB_ERR_NULL_DST); + return; + } + if (src_ptr_array[i] == NULL) { + imb_set_errno(state, IMB_ERR_NULL_SRC); + return; + } + } +#endif /* SAFE_PARAM */ + + switch (key_size) { + case IMB_KEY_128_BYTES: + state->aes_ecb_128_quic(src_ptr_array, exp_key_data, + dst_ptr_array, num_packets); + break; + case IMB_KEY_256_BYTES: + state->aes_ecb_256_quic(src_ptr_array, exp_key_data, + dst_ptr_array, num_packets); + break; + /* AES-192 is not supported by QUIC */ + case IMB_KEY_192_BYTES: + default: + imb_set_errno(state, IMB_ERR_KEY_LEN); + return; + }; + + imb_set_errno(state, 0); +} diff --git a/lib/x86_64/save_xmms.asm b/lib/x86_64/save_xmms.asm index b6a79903187696874a6fda468b1f0968db9ef801..722f91e3f2a14c0411e8bbc174af2d3b6aa835f0 100644 --- a/lib/x86_64/save_xmms.asm +++ b/lib/x86_64/save_xmms.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2012-2022, Intel Corporation +;; Copyright (c) 2012-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" %ifdef LINUX %define ARG1 rdi diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index 267c650c9eab0f5614ff79ef1a6f2ca99364247c..3e22b47fdba44be29c4e161b3f8f5807528df0d7 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -32,6 +32,8 @@ #include "ipsec-mb.h" #include "arch_x86_64.h" +#ifndef NO_SELF_TEST_DEV + static int process_job(IMB_MGR *p_mgr) { IMB_JOB *job = IMB_SUBMIT_JOB(p_mgr); @@ -45,7 +47,7 @@ static int process_job(IMB_MGR *p_mgr) /* flush to get the job processed */ job = IMB_FLUSH_JOB(p_mgr); - + /* if flush returns nothing then it's an error */ if (!job) return 0; @@ -230,6 +232,111 @@ static const uint8_t aes_ctr_256_cipher_text[] = { 0x56, 0x08, 0x63, 0xDC, 0x71, 0xE3, 0xE0, 0xC0, }; +/* + * AES-ECB test vectors + */ + +static const uint8_t aes_ecb_128_key[] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c +}; +static const uint8_t aes_ecb_128_plain_text[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; +static const uint8_t aes_ecb_128_cipher_text[] = { + 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60, + 0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97, + 0xf5, 0xd3, 0xd5, 0x85, 0x03, 0xb9, 0x69, 0x9d, + 0xe7, 0x85, 0x89, 0x5a, 0x96, 0xfd, 0xba, 0xaf, + 0x43, 0xb1, 0xcd, 0x7f, 0x59, 0x8e, 0xce, 0x23, + 0x88, 0x1b, 0x00, 0xe3, 0xed, 0x03, 0x06, 0x88, + 0x7b, 0x0c, 0x78, 0x5e, 0x27, 0xe8, 0xad, 0x3f, + 0x82, 0x23, 0x20, 0x71, 0x04, 0x72, 0x5d, 0xd4 +}; + +static const uint8_t aes_ecb_192_key[] = { + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, + 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b +}; +static const uint8_t aes_ecb_192_plain_text[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; +static const uint8_t aes_ecb_192_cipher_text[] = { + 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f, + 0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc, + 0x97, 0x41, 0x04, 0x84, 0x6d, 0x0a, 0xd3, 0xad, + 0x77, 0x34, 0xec, 0xb3, 0xec, 0xee, 0x4e, 0xef, + 0xef, 0x7a, 0xfd, 0x22, 0x70, 0xe2, 0xe6, 0x0a, + 0xdc, 0xe0, 0xba, 0x2f, 0xac, 0xe6, 0x44, 0x4e, + 0x9a, 0x4b, 0x41, 0xba, 0x73, 0x8d, 0x6c, 0x72, + 0xfb, 0x16, 0x69, 0x16, 0x03, 0xc1, 0x8e, 0x0e +}; + +static const uint8_t aes_ecb_256_key[] = { + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, + 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, + 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 +}; +static const uint8_t aes_ecb_256_plain_text[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; +static const uint8_t aes_ecb_256_cipher_text[] = { + 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c, + 0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8, + 0x59, 0x1c, 0xcb, 0x10, 0xd4, 0x10, 0xed, 0x26, + 0xdc, 0x5b, 0xa7, 0x4a, 0x31, 0x36, 0x28, 0x70, + 0xb6, 0xed, 0x21, 0xb9, 0x9c, 0xa6, 0xf4, 0xf9, + 0xf1, 0x53, 0xe7, 0xb1, 0xbe, 0xaf, 0xed, 0x1d, + 0x23, 0x30, 0x4b, 0x7a, 0x39, 0xf9, 0xf3, 0xff, + 0x06, 0x7d, 0x8d, 0x8f, 0x9e, 0x24, 0xec, 0xc7 +}; +static const uint8_t null_iv[] = { 0x00 }; + +/* + * Triple DES test vector TDES_EDE_CBC + */ +static const uint8_t tdes_ede_cbc_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 +}; + +static const uint8_t tdes_ede_cbc_iv[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 +}; + +static const uint8_t tdes_ede_cbc_plain_text[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; + +static const uint8_t tdes_ede_cbc_cipher_text[] = { + 0x89, 0x4b, 0xc3, 0x08, 0x54, 0x26, 0xa4, 0x41, + 0xf2, 0x7f, 0x73, 0xae, 0x26, 0xab, 0xbf, 0x74 +}; #define ADD_CIPHER_VECTOR(_cmode,_key,_iv,_plain,_cipher) \ {_cmode, _key, sizeof(_key), _iv, sizeof(_iv), \ @@ -248,13 +355,33 @@ struct self_test_cipher_vector cipher_vectors[] = { aes_ctr_192_plain_text, aes_ctr_192_cipher_text), ADD_CIPHER_VECTOR(IMB_CIPHER_CNTR, aes_ctr_256_key, aes_ctr_256_iv, aes_ctr_256_plain_text, aes_ctr_256_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_ECB, aes_ecb_128_key, null_iv, + aes_ecb_128_plain_text, aes_ecb_128_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_ECB, aes_ecb_192_key, null_iv, + aes_ecb_192_plain_text, aes_ecb_192_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_ECB, aes_ecb_256_key, null_iv, + aes_ecb_256_plain_text, aes_ecb_256_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_DES3, tdes_ede_cbc_key, tdes_ede_cbc_iv, + tdes_ede_cbc_plain_text, tdes_ede_cbc_cipher_text), }; +#define DES_KEY_SCHED_WORDS (IMB_DES_KEY_SCHED_SIZE / sizeof(uint64_t)) + static int self_test_ciphers(IMB_MGR *p_mgr) { + union { + struct { + DECLARE_ALIGNED(uint32_t expkey_enc[4*15], 16); + DECLARE_ALIGNED(uint32_t expkey_dec[4*15], 16); + } aes; + struct { + uint64_t key_sched1[DES_KEY_SCHED_WORDS]; + uint64_t key_sched2[DES_KEY_SCHED_WORDS]; + uint64_t key_sched3[DES_KEY_SCHED_WORDS]; + void *keys[3]; + } tdes; + } ks; uint8_t scratch[256]; - DECLARE_ALIGNED(uint32_t expkey_enc[4*15], 16); - DECLARE_ALIGNED(uint32_t expkey_dec[4*15], 16); unsigned i; while (IMB_FLUSH_JOB(p_mgr) != NULL) @@ -269,22 +396,39 @@ static int self_test_ciphers(IMB_MGR *p_mgr) if (v->plain_text_size > sizeof(scratch)) return 0; - switch (v->cipher_key_size) { - case IMB_KEY_128_BYTES: - IMB_AES_KEYEXP_128(p_mgr, v->cipher_key, - expkey_enc, expkey_dec); - break; - case IMB_KEY_192_BYTES: - IMB_AES_KEYEXP_192(p_mgr, v->cipher_key, - expkey_enc, expkey_dec); - break; - case IMB_KEY_256_BYTES: - IMB_AES_KEYEXP_256(p_mgr, v->cipher_key, - expkey_enc, expkey_dec); - break; - default: - /* invalid key size */ - return 0; + if (v->cipher_mode == IMB_CIPHER_DES3) { + if (v->cipher_key_size != IMB_KEY_192_BYTES) { + /* invalid key size */ + return 0; + } + des_key_schedule(ks.tdes.key_sched1, &v->cipher_key[0]); + des_key_schedule(ks.tdes.key_sched2, &v->cipher_key[8]); + des_key_schedule(ks.tdes.key_sched3, + &v->cipher_key[16]); + ks.tdes.keys[0] = ks.tdes.key_sched1; + ks.tdes.keys[1] = ks.tdes.key_sched2; + ks.tdes.keys[2] = ks.tdes.key_sched3; + } else { + switch (v->cipher_key_size) { + case IMB_KEY_128_BYTES: + IMB_AES_KEYEXP_128(p_mgr, v->cipher_key, + ks.aes.expkey_enc, + ks.aes.expkey_dec); + break; + case IMB_KEY_192_BYTES: + IMB_AES_KEYEXP_192(p_mgr, v->cipher_key, + ks.aes.expkey_enc, + ks.aes.expkey_dec); + break; + case IMB_KEY_256_BYTES: + IMB_AES_KEYEXP_256(p_mgr, v->cipher_key, + ks.aes.expkey_enc, + ks.aes.expkey_dec); + break; + default: + /* invalid key size */ + return 0; + } } /* test encrypt direction */ @@ -296,12 +440,18 @@ static int self_test_ciphers(IMB_MGR *p_mgr) job->src = v->plain_text; job->dst = scratch; job->cipher_mode = v->cipher_mode; - job->enc_keys = expkey_enc; - if (v->cipher_mode != IMB_CIPHER_CNTR) - job->dec_keys = expkey_dec; + if (v->cipher_mode == IMB_CIPHER_DES3) { + job->enc_keys = ks.tdes.keys; + } else { + job->enc_keys = ks.aes.expkey_enc; + if (v->cipher_mode != IMB_CIPHER_CNTR) + job->dec_keys = ks.aes.expkey_dec; + } job->key_len_in_bytes = v->cipher_key_size; - job->iv = v->cipher_iv; - job->iv_len_in_bytes = v->cipher_iv_size; + if (v->cipher_mode != IMB_CIPHER_ECB) { + job->iv = v->cipher_iv; + job->iv_len_in_bytes = v->cipher_iv_size; + } job->cipher_start_src_offset_in_bytes = 0; job->msg_len_to_cipher_in_bytes = v->plain_text_size; @@ -324,12 +474,18 @@ static int self_test_ciphers(IMB_MGR *p_mgr) job->src = v->cipher_text; job->dst = scratch; job->cipher_mode = v->cipher_mode; - job->dec_keys = expkey_dec; - if (v->cipher_mode == IMB_CIPHER_CNTR) - job->enc_keys = expkey_enc; + if (v->cipher_mode == IMB_CIPHER_DES3) { + job->dec_keys = ks.tdes.keys; + } else { + job->dec_keys = ks.aes.expkey_dec; + if (v->cipher_mode == IMB_CIPHER_CNTR) + job->enc_keys = ks.aes.expkey_enc; + } job->key_len_in_bytes = v->cipher_key_size; - job->iv = v->cipher_iv; - job->iv_len_in_bytes = v->cipher_iv_size; + if (v->cipher_mode != IMB_CIPHER_ECB) { + job->iv = v->cipher_iv; + job->iv_len_in_bytes = v->cipher_iv_size; + } job->cipher_start_src_offset_in_bytes = 0; job->msg_len_to_cipher_in_bytes = v->plain_text_size; @@ -342,7 +498,7 @@ static int self_test_ciphers(IMB_MGR *p_mgr) /* check for plain text mismatch */ if (memcmp(scratch, v->plain_text, v->plain_text_size)) return 0; - + } /* for(cipher_vectors) */ return 1; @@ -781,84 +937,18 @@ static int self_test_hash(IMB_MGR *p_mgr) job->msg_len_to_hash_in_bytes = v->message_size; job->auth_tag_output = scratch; job->auth_tag_output_len_in_bytes = v->tag_size; - - if (v->hash_mode == IMB_AUTH_HMAC_SHA_1) { - /* compute IPAD and OPAD */ - unsigned j; - - IMB_ASSERT(sizeof(scratch) >= IMB_SHA1_BLOCK_SIZE); - - memset(scratch, 0x36, IMB_SHA1_BLOCK_SIZE); - for (j = 0; j < v->hash_key_size; j++) - scratch[j] ^= v->hash_key[j]; - IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - memset(scratch, 0x5c, IMB_SHA1_BLOCK_SIZE); - for (j = 0; j < v->hash_key_size; j++) - scratch[j] ^= v->hash_key[j]; - IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_opad); - + if (v->hash_mode >= IMB_AUTH_HMAC_SHA_1 && + v->hash_mode <= IMB_AUTH_HMAC_SHA_512) { + imb_hmac_ipad_opad(p_mgr, v->hash_mode, v->hash_key, + v->hash_key_size, + hmac_ipad, hmac_opad); job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; } - - if (v->hash_mode == IMB_AUTH_HMAC_SHA_224 || - v->hash_mode == IMB_AUTH_HMAC_SHA_256) { - /* compute IPAD and OPAD */ - unsigned j; - - IMB_ASSERT(sizeof(scratch) >= IMB_SHA_256_BLOCK_SIZE); - - memset(scratch, 0x36, IMB_SHA_256_BLOCK_SIZE); - for (j = 0; j < v->hash_key_size; j++) - scratch[j] ^= v->hash_key[j]; - if (v->hash_mode == IMB_AUTH_HMAC_SHA_224) - IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - else - IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - memset(scratch, 0x5c, IMB_SHA_256_BLOCK_SIZE); - for (j = 0; j < v->hash_key_size; j++) - scratch[j] ^= v->hash_key[j]; - if (v->hash_mode == IMB_AUTH_HMAC_SHA_224) - IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_opad); - else - IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_opad); - - job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; - job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; - } - - if (v->hash_mode == IMB_AUTH_HMAC_SHA_384 || - v->hash_mode == IMB_AUTH_HMAC_SHA_512) { - /* compute IPAD and OPAD */ - unsigned j; - - IMB_ASSERT(sizeof(scratch) >= IMB_SHA_512_BLOCK_SIZE); - - memset(scratch, 0x36, IMB_SHA_512_BLOCK_SIZE); - for (j = 0; j < v->hash_key_size; j++) - scratch[j] ^= v->hash_key[j]; - if (v->hash_mode == IMB_AUTH_HMAC_SHA_384) - IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - else - IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - memset(scratch, 0x5c, IMB_SHA_512_BLOCK_SIZE); - for (j = 0; j < v->hash_key_size; j++) - scratch[j] ^= v->hash_key[j]; - if (v->hash_mode == IMB_AUTH_HMAC_SHA_384) - IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_opad); - else - IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_opad); - - job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; - job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; - } - if (v->hash_mode == IMB_AUTH_AES_CMAC) { IMB_AES_KEYEXP_128(p_mgr, v->hash_key, expkey, dust); - IMB_AES_CMAC_SUBKEY_GEN_128(p_mgr, expkey, skey1, skey2); + IMB_AES_CMAC_SUBKEY_GEN_128(p_mgr, expkey, + skey1, skey2); job->u.CMAC._key_expanded = expkey; job->u.CMAC._skey1 = skey1; job->u.CMAC._skey2 = skey2; @@ -866,7 +956,8 @@ static int self_test_hash(IMB_MGR *p_mgr) if (v->hash_mode == IMB_AUTH_AES_CMAC_256) { IMB_AES_KEYEXP_256(p_mgr, v->hash_key, expkey, dust); - IMB_AES_CMAC_SUBKEY_GEN_256(p_mgr, expkey, skey1, skey2); + IMB_AES_CMAC_SUBKEY_GEN_256(p_mgr, expkey, + skey1, skey2); job->u.CMAC._key_expanded = expkey; job->u.CMAC._skey1 = skey1; job->u.CMAC._skey2 = skey2; @@ -878,7 +969,7 @@ static int self_test_hash(IMB_MGR *p_mgr) job->u.GMAC._iv = v->hash_iv; job->u.GMAC.iv_len_in_bytes = v->hash_iv_size; } - + if (v->hash_mode == IMB_AUTH_AES_GMAC_192) { IMB_AES192_GCM_PRE(p_mgr, v->hash_key, &gmac_key); job->u.GMAC._key = &gmac_key; @@ -1563,6 +1654,8 @@ static int self_test_aead(IMB_MGR *p_mgr) return 1; } +#endif /* NO_SELF_TEST_DEV */ + /* * ============================================================================= * SELF-TEST INTERNAL API @@ -1573,6 +1666,10 @@ IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) { int ret = 1; +#ifdef NO_SELF_TEST_DEV + p_mgr->features &= ~(IMB_FEATURE_SELF_TEST | + IMB_FEATURE_SELF_TEST_PASS); +#else p_mgr->features |= IMB_FEATURE_SELF_TEST; p_mgr->features &= ~IMB_FEATURE_SELF_TEST_PASS; @@ -1588,10 +1685,6 @@ IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) if (ret) p_mgr->features |= IMB_FEATURE_SELF_TEST_PASS; -#ifdef NO_SELF_TEST_DEV - p_mgr->features &= ~(IMB_FEATURE_SELF_TEST | - IMB_FEATURE_SELF_TEST_PASS); - ret = 1; #endif return ret; diff --git a/lib/x86_64/snow3g_iv.c b/lib/x86_64/snow3g_iv.c index 09dddeac7d95cb9fa094c2ac814668b3451d7280..6bf51466906d554f3aa68d03245593ce96685218 100644 --- a/lib/x86_64/snow3g_iv.c +++ b/lib/x86_64/snow3g_iv.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -82,10 +82,14 @@ snow3g_f9_iv_gen(const uint32_t count, const uint32_t fresh, * and reversed to generate the IV in Big Endian format */ /* IV[3] = FRESH ^ (DIRECTION[0] << 17) */ - iv32[3] = bswap4(fresh ^ (dir << 15)); + const uint32_t dir_b15 = dir ? (1UL << 15) : 0; /* IV[2] = DIRECTION[0] ^ COUNT[0-31] */ - iv32[2] = bswap4(count ^ (dir << 31)); + const uint32_t dir_b31 = dir ? (1UL << 31) : 0; + + iv32[3] = bswap4(fresh ^ dir_b15); + + iv32[2] = bswap4(count ^ dir_b31); /* IV[1] = FRESH */ iv32[1] = bswap4(fresh); diff --git a/lib/x86_64/snow3g_tables.c b/lib/x86_64/snow3g_tables.c index 12d4c2f516e50e141ad33cc5a37ed984d99ac90d..c8888e57b6d2ef4217780641fe252b981aaf9351 100644 --- a/lib/x86_64/snow3g_tables.c +++ b/lib/x86_64/snow3g_tables.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/x86_64/version.c b/lib/x86_64/version.c index f86c87ef558f57bb4259c45b3843d6f939297f23..b549bdeb9913ca975653738b8272bdf0b87355ec 100644 --- a/lib/x86_64/version.c +++ b/lib/x86_64/version.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/lib/x86_64/wireless_common.asm b/lib/x86_64/wireless_common.asm index b55a9fe36ffcdd6c4eceee0f349e2369951a949a..24e075798069201e08490a99dcb3b8cd335ed58c 100644 --- a/lib/x86_64/wireless_common.asm +++ b/lib/x86_64/wireless_common.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2019-2022, Intel Corporation +;; Copyright (c) 2019-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" +%include "include/os.inc" mksection .rodata default rel diff --git a/lib/x86_64/zuc_common.asm b/lib/x86_64/zuc_common.asm index 19961b3613b98eead10fe3a0277820ff0b04e348..e811ae683edae2da7f99846fc54123d1f17d73fd 100644 --- a/lib/x86_64/zuc_common.asm +++ b/lib/x86_64/zuc_common.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2009-2022, Intel Corporation +;; Copyright (c) 2009-2023, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,8 +25,8 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%include "include/os.asm" -%include "include/reg_sizes.asm" +%include "include/os.inc" +%include "include/reg_sizes.inc" %include "include/zuc_sbox.inc" mksection .rodata diff --git a/lib/x86_64/zuc_iv.c b/lib/x86_64/zuc_iv.c index c4c61c355c0345a0d0f3624a7140690db953f6a3..6d42000c6fab113028998264dd692cd14b99616f 100644 --- a/lib/x86_64/zuc_iv.c +++ b/lib/x86_64/zuc_iv.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/mkdep.bat b/mkdep.bat index 617ab06097763e1d072b8e00a63b270b2cec6285..ad39d14be1dc09723e6b93721df83461fbf0faac 100644 --- a/mkdep.bat +++ b/mkdep.bat @@ -1,5 +1,5 @@ @echo off -REM // Copyright (c) 2020-2022, Intel Corporation +REM // Copyright (c) 2020-2023, Intel Corporation REM // REM // Redistribution and use in source and binary forms, with or without REM // modification, are permitted provided that the following conditions are met: diff --git a/perf/CMakeLists.txt b/perf/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..59be4a2c7c34fae7902a01c62732d2a780bed5bf --- /dev/null +++ b/perf/CMakeLists.txt @@ -0,0 +1,94 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +######################################## +# set app and library names +######################################## +set(PERF_APP imb-perf) +set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../lib) + +######################################## +# set source files +######################################## +set(PERF_APP_SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/ipsec_perf.c + ${CMAKE_CURRENT_SOURCE_DIR}/msr.c +) + +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(PERF_APP_SRC_FILES_ASM + ${CMAKE_CURRENT_SOURCE_DIR}/misc_aarch64.S + ) +else() + set(PERF_APP_SRC_FILES_ASM + ${CMAKE_CURRENT_SOURCE_DIR}/misc.asm + ) +endif() + +######################################## +# add target +######################################## +add_executable(${PERF_APP} ${PERF_APP_SRC_FILES} ${PERF_APP_SRC_FILES_ASM}) + +######################################## +# setup NASM +######################################## +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + enable_language(ASM) + if(NOT CMAKE_ASM_COMPILER_LOADED) + message(FATAL_ERROR "Can't find assembler") + endif() +else() + enable_language(ASM_NASM) + if(NOT CMAKE_ASM_NASM_COMPILER_LOADED) + message(FATAL_ERROR "Can't find assembler") + endif() +endif() +set(CAN_USE_ASSEMBLER TRUE) + +######################################## +# set C compiler and NASM options +######################################## +# add OS specific options +if(WINDOWS) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/windows.cmake) +else() + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/unix.cmake) +endif() + +######################################## +# extra target properties +######################################## +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${LIB_DIR}) +link_directories(${LIB_DIR}) + +target_link_libraries(${PERF_APP} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${PERF_APP} PRIVATE ${APP_DEFINES}) + diff --git a/perf/Makefile b/perf/Makefile index c6e286ae89d3f5173f44d0b3da50d51d901c2521..abc138584f85f2a1ca63e06957169f2724a73fed 100644 --- a/perf/Makefile +++ b/perf/Makefile @@ -1,8 +1,8 @@ -# Copyright (c) 2017-2022, Intel Corporation -# +# Copyright (c) 2017-2023, Intel Corporation +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright @@ -11,7 +11,7 @@ # * Neither the name of Intel Corporation nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -23,23 +23,33 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -EXE=ipsec_perf -INSTPATH ?= /usr/include/ipsec-mb.h -LIB_DIR ?= ../lib - ARCH = $(shell uname -m) +EXE=imb-perf +INSTPATH ?= /usr/include/ipsec-mb.h +LIB_DIR ?= ../lib NASM ?= nasm MINGW ?= $(shell $(CC) -dM -E - < /dev/null | grep -i mingw | wc -l | sed 's/^ *//') -CFLAGS = -D_GNU_SOURCE -DNO_COMPAT_IMB_API_053 $(INCLUDES) \ +ifeq ($(ARCH),x86_64) +CFLAGS = -D_GNU_SOURCE $(INCLUDES) \ + -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ + -Wcast-qual -Wundef -Wwrite-strings \ + -Wformat -Wformat-security \ + -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ + -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ + -pthread -fno-delete-null-pointer-checks -fwrapv -std=c99 +else +CFLAGS = -D_GNU_SOURCE $(INCLUDES) \ -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ -Wcast-qual -Wundef -Wwrite-strings \ -Wformat -Wformat-security \ -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ - -pthread -fno-delete-null-pointer-checks -fwrapv + -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function \ + -pthread -fno-delete-null-pointer-checks -fwrapv -std=c99 +endif # -fno-strict-overflow is not supported by clang ifneq ($(CC),clang) @@ -55,28 +65,25 @@ endif # if "-z ibt" is supported then assume "-z shstk, -z cet-report=error" are also supported # "-fcf-protection" needs to be checked separately -ifeq ($(ARCH),x86_64) +ifneq ($(ARCH),aarch64) ifeq ($(MINGW),0) CC_HAS_CET = $(and $(shell $(CC) --target-help 2> /dev/null | grep -m1 -e "-z ibt" | wc -l), \ $(shell $(CC) --help=common 2> /dev/null | grep -m1 -e "-fcf-protection" | wc -l)) CET_LDFLAGS=-r -z ibt -z shstk -endif # MINGW -endif # x86_64 - -ifeq ($(ARCH),x86_64) +endif ifeq ($(CC_HAS_CET),1) CFLAGS += -fcf-protection=full endif -endif +endif # aarch64 ifeq ($(MINGW),0) LDFLAGS = -fPIE -z noexecstack -z relro -z now -pthread endif -ifeq ($(ARCH),x86_64) +ifneq ($(ARCH),aarch64) ifeq ($(CC_HAS_CET),1) LDFLAGS += -fcf-protection=full -Wl,-z,ibt -Wl,-z,shstk -Wl,-z,cet-report=error endif -endif +endif # aarch64 LDLIBS = -lIPSec_MB ifeq ("$(shell test -e $(INSTPATH) && echo -n yes)","yes") @@ -84,7 +91,7 @@ ifeq ("$(shell test -e $(INSTPATH) && echo -n yes)","yes") CFLAGS += else # library not installed -CFLAGS += -I../lib/include -I../lib +CFLAGS += -I../lib LDFLAGS += -L$(LIB_DIR) endif @@ -97,18 +104,20 @@ ifeq ($(MINGW),0) CFLAGS += -O3 -fPIE -fstack-protector -D_FORTIFY_SOURCE=2 else CFLAGS += -O2 -fPIE -endif # MINGW -endif # DEBUG +endif +endif SOURCES := ipsec_perf.c msr.c ifneq ($(ARCH),aarch64) ASM_SOURCES := misc.asm OBJECTS := $(SOURCES:%.c=%.o) $(ASM_SOURCES:%.asm=%.o) +DEPFILES := $(OBJECTS:%.o=%.d) else ASM_SOURCES := misc_aarch64.S CFLAGS += -march=armv8-a OBJECTS := $(SOURCES:%.c=%.o) $(ASM_SOURCES:%.S=%.o) -endif # AARCH64 +DEPFILES := $(OBJECTS:%.o=%.d) +endif # aarch64 CHECKPATCH ?= checkpatch.pl CPPCHECK ?= cppcheck @@ -122,30 +131,45 @@ ifneq ($(ARCH),aarch64) ifeq ($(CC_HAS_CET),1) $(LD) $(CET_LDFLAGS) -o $@.tmp $@ mv $@.tmp $@ -endif # CC_HAS_CET +endif else %.o:%.S $(CC) -c $(CFLAGS) $< -o $@ -endif # AARCH64 +endif # aarch64 all: $(EXE) $(EXE): $(OBJECTS) $(CC) $(LDFLAGS) $^ $(LDLIBS) -o $@ -ipsec_perf.o: $(SOURCES) - .PHONY: clean clean: -rm -f $(OBJECTS) + -rm -f $(DEPFILES) -rm -f $(EXE) -SOURCES_STYLE := $(foreach infile,$(SOURCES),-f $(infile)) -CHECKPATCH?=checkpatch.pl +CLANGFORMAT?=clang-format .PHONY: style style: - $(CHECKPATCH) --no-tree --no-signoff --emacs --no-color \ ---ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,\ -UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,\ -COMPLEX_MACRO,SPACING,STORAGE_CLASS,USE_FUNC,NEW_TYPEDEFS,VOLATILE,\ -CONSTANT_COMPARISON $(SOURCES_STYLE) + @for file in $(wildcard *.[ch]); do \ + echo "Checking style $$file"; \ + $(CLANGFORMAT) -style=file "$$file" | diff "$$file" - | tee /dev/stderr | [ $$(wc -c) -eq 0 ] || \ + { echo "ERROR: $$file has style problems"; exit 1; } \ + done + +# cppcheck analysis check +CPPCHECK ?= cppcheck +CPPCHECK_OPTS ?= -I../lib -I./ +CPPCHECK_FLAGS ?= -j $(shell getconf _NPROCESSORS_ONLN) +CPPCHECK_FLAGS1 ?= --cppcheck-build-dir=.cppcheck $(CPPCHECK_FLAGS) +CPPCHECK_FLAGS2 ?= --cppcheck-build-dir=.bughunt $(CPPCHECK_FLAGS) + +.PHONY: cppcheck +cppcheck: + mkdir -p .cppcheck + $(CPPCHECK) --force --enable=all $(CPPCHECK_FLAGS1) $(CPPCHECK_OPTS) ./ + +.PHONY: bughunt +bughunt: + mkdir -p .bughunt + $(CPPCHECK) --bug-hunting --inconclusive $(CPPCHECK_FLAGS2) $(CPPCHECK_OPTS) ./ diff --git a/perf/README b/perf/README index ee7d63220bfc812c60b270f591b2f83e560eb040..e3095fca5f6fde919a3ff2bcd293fe08fd34d277 100644 --- a/perf/README +++ b/perf/README @@ -1,5 +1,5 @@ ======================================================================== -README for Multi-Buffer Crypto for IPsec Library API +README for Intel(R) Multi-Buffer Crypto for IPsec Library API performance measurement tool February 2017 @@ -58,10 +58,10 @@ Usage ===== You can simply check list of arguments by typing: - ./ipsec_perf -h + ./imb-perf -h Usage example: - ./ipsec_perf -c --no-avx512 --no-gcm -o 24 + ./imb-perf -c --no-avx512 --no-gcm -o 24 Later you can pass output to ipsec_diff_tool.py for data analysis: diff --git a/perf/cmake/unix.cmake b/perf/cmake/unix.cmake new file mode 100644 index 0000000000000000000000000000000000000000..3db98a36c011a3e7922e39052cd3cae56ef4e15d --- /dev/null +++ b/perf/cmake/unix.cmake @@ -0,0 +1,68 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# Performance application CMake Unix config +# ############################################################################## + +set(IPSEC_MB_LIB IPSec_MB) + +# set NASM flags +set(CMAKE_ASM_NASM_FLAGS "-Werror -felf64 -Xgnu -gdwarf -DLINUX -D__linux__") + +# set compiler definitions +set(APP_DEFINES LINUX _GNU_SOURCE) + +# set C compiler flags +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(CMAKE_C_FLAGS + "-W -Wall -Wextra -Wmissing-declarations \ +-Wpointer-arith -Wcast-qual -Wundef -Wwrite-strings -Wformat \ +-Wformat-security -Wunreachable-code -Wmissing-noreturn -Wsign-compare \ +-Wno-endif-labels -Wstrict-prototypes -Wmissing-prototypes \ +-Wold-style-definition -fno-delete-null-pointer-checks -fwrapv -std=c99 \ +-Wno-unused-parameter -Wno-unused-variable -Wno-unused-function") +else() + set(CMAKE_C_FLAGS + "-W -Wall -Wextra -Wmissing-declarations \ +-Wpointer-arith -Wcast-qual -Wundef -Wwrite-strings -Wformat \ +-Wformat-security -Wunreachable-code -Wmissing-noreturn -Wsign-compare \ +-Wno-endif-labels -Wstrict-prototypes -Wmissing-prototypes \ +-Wold-style-definition -fno-delete-null-pointer-checks -fwrapv -std=c99") +endif() +set(CMAKE_C_FLAGS_DEBUG "-g -O0 -DDEBUG") +set(CMAKE_C_FLAGS_RELEASE "-O3 -fPIE -fstack-protector -D_FORTIFY_SOURCE=2") +set(CMAKE_EXE_LINKER_FLAGS "-fPIE -z noexecstack -z relro -z now -pthread") +set(CMAKE_EXE_LINKER_FLAGS_DEBUG "-g") +# -fno-strict-overflow is not supported by clang +if(CMAKE_COMPILER_IS_GNUCC) + string(APPEND CMAKE_C_FLAGS " -fno-strict-overflow") +endif() + +# copy perf scripts to binary directory +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ipsec_diff_tool.py + ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ipsec_perf_tool.py + ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) diff --git a/perf/cmake/windows.cmake b/perf/cmake/windows.cmake new file mode 100644 index 0000000000000000000000000000000000000000..4117bf69a80e4fba9cf2837ae25e595cb13ec46b --- /dev/null +++ b/perf/cmake/windows.cmake @@ -0,0 +1,60 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# Performance application CMake Windows config +# ############################################################################## + +set(IPSEC_MB_LIB libIPSec_MB) + +# set NASM flags +set(CMAKE_ASM_NASM_FLAGS "-Werror -fwin64 -Xvc -DWIN_ABI") + +if(WINRING0_DIR) + string(APPEND EXTRA_CFLAGS " /DWIN_MSR -I ${WINRING0_DIR}") +endif() + +# set C compiler flags +set(CMAKE_C_FLAGS + "/nologo /D_CRT_SECURE_NO_WARNINGS /Y- /W3 /WX- /Gm- /fp:precise /EHsc /std:c11 ${EXTRA_CFLAGS}" +) +set(CMAKE_C_FLAGS_DEBUG "/Od /DDEBUG /Z7") +set(CMAKE_C_FLAGS_RELEASE "/O2 /Oi") +set(CMAKE_EXE_LINKER_FLAGS_DEBUG "/debug") + +# copy perf scripts to binary directory +add_custom_command( + TARGET ${PERF_APP} + POST_BUILD + COMMAND + ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ipsec_diff_tool.py" + $) +add_custom_command( + TARGET ${PERF_APP} + POST_BUILD + COMMAND + ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ipsec_perf_tool.py" + $) + diff --git a/perf/ipsec_diff_tool.py b/perf/ipsec_diff_tool.py index d21b049f2424ebbdeafe55e929e78d2d5f1bc0d3..6737cf532768c1892454937ba9a8fe982f605df8 100755 --- a/perf/ipsec_diff_tool.py +++ b/perf/ipsec_diff_tool.py @@ -2,7 +2,7 @@ """ ********************************************************************** - Copyright(c) 2017-2022, Intel Corporation All rights reserved. + Copyright(c) 2017-2023, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -327,7 +327,7 @@ class DiffTool(object): print("\t-c - takes packet size as argument and then it will calculate cycle cost") print("\t-t - takes packet size and clock speed as arguments and then it will calculate throughput in Mbps") print("\t-s - calculates the slope and intercept") - print("\tfile_a, file_b - text files containing output from ipsec_perf tool") + print("\tfile_a, file_b - text files containing output from imb-perf tool") print("\ttol - tolerance [%], must be >= 0, default 5\n") print("Examples:") print("\tdefault no arguments prints slope and intercept") diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 55d392aeb7b602a55ed03b4295f1f199e8d5320c..b9673e761513e35c1c9781ce6e88d4bd7586cf1f 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -27,6 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ +#include #include #include #include @@ -37,6 +38,11 @@ #include #endif +#ifdef _MSC_VER +/* disable C5105 warning produced by standard headers C11 C standard */ +#pragma warning(disable : 5105) +#endif + #ifdef _WIN32 #include /* memalign() or _aligned_malloc()/aligned_free() */ #include @@ -47,21 +53,21 @@ #define __forceinline static __forceinline #define __func__ __FUNCTION__ #define strcasecmp _stricmp -#else /* _WIN32 */ +#else #include -#ifdef __x86_64__ +#ifndef __aarch64__ #include -#endif /* __x86_64__ */ +#endif #define __forceinline static inline __attribute__((always_inline)) #include #include -#if defined (__FreeBSD__) +#if defined(__FreeBSD__) #include typedef cpuset_t cpu_set_t; -#else /* __FreeBSD__ */ +#else #include -#endif /* __FreeBSD__ */ -#endif /* _WIN32 */ +#endif +#endif #include @@ -83,20 +89,19 @@ typedef cpuset_t cpu_set_t; /* max offset applied to a buffer - this is to avoid collisions in L1 */ #define MAX_BUFFER_OFFSET 4096 /* max value of sha_size_incr */ -#define MAX_SHA_SIZE_INCR 128 +#define MAX_SHA_SIZE_INCR 128 /* region size for one buffer rounded up to 4K page size */ -#define REGION_SIZE (((JOB_SIZE_TOP + (MAX_BUFFER_OFFSET + \ - MAX_SHA_SIZE_INCR)) + 4095) & (~4095)) +#define REGION_SIZE (((JOB_SIZE_TOP + (MAX_BUFFER_OFFSET + MAX_SHA_SIZE_INCR)) + 4095) & (~4095)) /* number of test buffers */ #define NUM_OFFSETS (BUFSIZE / REGION_SIZE) -#define NUM_RUNS 16 +#define NUM_RUNS 16 /* maximum number of 128-bit expanded keys */ #define KEYS_PER_JOB 15 -#define AAD_SIZE_MAX JOB_SIZE_TOP -#define CCM_AAD_SIZE_MAX 46 -#define DEFAULT_GCM_AAD_SIZE 12 -#define DEFAULT_CCM_AAD_SIZE 8 +#define AAD_SIZE_MAX JOB_SIZE_TOP +#define CCM_AAD_SIZE_MAX 46 +#define DEFAULT_GCM_AAD_SIZE 12 +#define DEFAULT_CCM_AAD_SIZE 8 #define DEFAULT_CHACHA_POLY_AAD_SIZE 12 #define DEFAULT_SNOW_V_AEAD_AAD_SIZE 16 @@ -104,8 +109,8 @@ typedef cpuset_t cpu_set_t; #define ITER_SCALE_SHORT 200000 #define ITER_SCALE_LONG 2000000 -#define BITS(x) (sizeof(x) * 8) -#define DIM(x) (sizeof(x)/sizeof(x[0])) +#define BITS(x) (sizeof(x) * 8) +#define DIM(x) (sizeof(x) / sizeof(x[0])) #define DIV_ROUND_UP(x, y) ((x + y - 1) / y) #define MAX_NUM_THREADS 16 /* Maximum number of threads that can be created */ @@ -115,25 +120,21 @@ typedef cpuset_t cpu_set_t; #define IA32_MSR_CPU_UNHALTED_THREAD 0x30A #define DEFAULT_BURST_SIZE 32 -#define MAX_BURST_SIZE 256 - -enum arch_type_e { - ARCH_SSE = 0, - ARCH_AVX, - ARCH_AVX2, - ARCH_AVX512, - ARCH_AARCH64, - ARCH_SVE256, - NUM_ARCHS -}; +#define MAX_BURST_SIZE 256 + +#ifndef __aarch64__ +enum arch_type_e { ARCH_SSE = 0, ARCH_AVX, ARCH_AVX2, ARCH_AVX512, NUM_ARCHS }; +#else +enum arch_type_e { ARCH_AARCH64 = 0, ARCH_SVE256, NUM_ARCHS }; +#endif /* This enum will be mostly translated to IMB_CIPHER_MODE * (make sure to update c_mode_names list in print_times function) */ enum test_cipher_mode_e { TEST_CBC = 1, TEST_CNTR, - TEST_CNTR8, /* CNTR with increased buffer by 8 */ - TEST_CNTR_BITLEN, /* CNTR-BITLEN */ + TEST_CNTR8, /* CNTR with increased buffer by 8 */ + TEST_CNTR_BITLEN, /* CNTR-BITLEN */ TEST_CNTR_BITLEN4, /* CNTR-BITLEN with 4 less bits in the last byte */ TEST_ECB, TEST_CBCS_1_9, @@ -142,7 +143,7 @@ enum test_cipher_mode_e { TEST_AESDOCSIS8, /* AES DOCSIS with increased buffer size by 8 */ TEST_DESDOCSIS, TEST_DESDOCSIS4, /* DES DOCSIS with increased buffer size by 4 */ - TEST_GCM, /* Additional field used by GCM, not translated */ + TEST_GCM, /* Additional field used by GCM, not translated */ TEST_CCM, TEST_DES, TEST_3DES, @@ -178,7 +179,7 @@ enum test_hash_alg_e { TEST_HASH_CMAC_256, TEST_NULL_HASH, TEST_DOCSIS_CRC32, - TEST_HASH_GCM, /* Additional field used by GCM, not translated */ + TEST_HASH_GCM, /* Additional field used by GCM, not translated */ TEST_CUSTOM_HASH, /* unused */ TEST_HASH_CCM, TEST_PON_CRC_BIP, @@ -210,287 +211,132 @@ enum test_hash_alg_e { /* Struct storing cipher parameters */ struct params_s { - IMB_CIPHER_DIRECTION cipher_dir; + IMB_CIPHER_DIRECTION cipher_dir; enum test_cipher_mode_e cipher_mode; - enum test_hash_alg_e hash_alg; - uint32_t aes_key_size; - uint32_t size_aes; - uint64_t aad_size; - uint32_t num_sizes; - uint32_t core; + enum test_hash_alg_e hash_alg; + uint32_t aes_key_size; + uint32_t size_aes; + uint64_t aad_size; + uint32_t num_sizes; + uint32_t core; }; struct custom_job_params { enum test_cipher_mode_e cipher_mode; - enum test_hash_alg_e hash_alg; - uint32_t aes_key_size; - IMB_CIPHER_DIRECTION cipher_dir; + enum test_hash_alg_e hash_alg; + uint32_t aes_key_size; + IMB_CIPHER_DIRECTION cipher_dir; }; union params { - enum arch_type_e arch_type; + enum arch_type_e arch_type; struct custom_job_params job_params; }; struct str_value_mapping { - const char *name; - union params values; + const char *name; + union params values; }; -const struct str_value_mapping arch_str_map[] = { - {.name = "SSE", .values.arch_type = ARCH_SSE }, - {.name = "AVX", .values.arch_type = ARCH_AVX }, - {.name = "AVX2", .values.arch_type = ARCH_AVX2 }, - {.name = "AVX512", .values.arch_type = ARCH_AVX512 }, - {.name = "AARCH64",.values.arch_type = ARCH_AARCH64 }, - {.name = "SVE256", .values.arch_type = ARCH_SVE256 }, -}; +#ifndef __aarch64__ +const struct str_value_mapping arch_str_map[] = { { .name = "SSE", .values.arch_type = ARCH_SSE }, + { .name = "AVX", .values.arch_type = ARCH_AVX }, + { .name = "AVX2", .values.arch_type = ARCH_AVX2 }, + { .name = "AVX512", + .values.arch_type = ARCH_AVX512 } }; +#else +const struct str_value_mapping arch_str_map[] = { { .name = "AARCH64", + .values.arch_type = ARCH_AARCH64 }, + { .name = "SVE256", + .values.arch_type = ARCH_SVE256 } }; +#endif const struct str_value_mapping cipher_algo_str_map[] = { -#ifdef __x86_64__ - { - .name = "aes-cbc-128", - .values.job_params = { - .cipher_mode = TEST_CBC, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-cbc-192", - .values.job_params = { - .cipher_mode = TEST_CBC, - .aes_key_size = IMB_KEY_192_BYTES - } - }, - { - .name = "aes-cbc-256", - .values.job_params = { - .cipher_mode = TEST_CBC, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-ctr-128", - .values.job_params = { - .cipher_mode = TEST_CNTR, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-ctr-192", - .values.job_params = { - .cipher_mode = TEST_CNTR, - .aes_key_size = IMB_KEY_192_BYTES - } - }, - { - .name = "aes-ctr-256", - .values.job_params = { - .cipher_mode = TEST_CNTR, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-ctr8-128", - .values.job_params = { - .cipher_mode = TEST_CNTR8, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-ctr8-192", - .values.job_params = { - .cipher_mode = TEST_CNTR8, - .aes_key_size = IMB_KEY_192_BYTES - } - }, - { - .name = "aes-ctr8-256", - .values.job_params = { - .cipher_mode = TEST_CNTR8, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-ctr-bit-128", - .values.job_params = { - .cipher_mode = TEST_CNTR_BITLEN, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-ctr-bit-192", - .values.job_params = { - .cipher_mode = TEST_CNTR_BITLEN, - .aes_key_size = IMB_KEY_192_BYTES - } - }, - { - .name = "aes-ctr-bit-256", - .values.job_params = { - .cipher_mode = TEST_CNTR_BITLEN, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-ctr-bit4-128", - .values.job_params = { - .cipher_mode = TEST_CNTR_BITLEN4, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-ctr-bit4-192", - .values.job_params = { - .cipher_mode = TEST_CNTR_BITLEN4, - .aes_key_size = IMB_KEY_192_BYTES - } - }, - { - .name = "aes-ctr-bit4-256", - .values.job_params = { - .cipher_mode = TEST_CNTR_BITLEN4, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-ecb-128", - .values.job_params = { - .cipher_mode = TEST_ECB, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-ecb-192", - .values.job_params = { - .cipher_mode = TEST_ECB, - .aes_key_size = IMB_KEY_192_BYTES - } - }, - { - .name = "aes-ecb-256", - .values.job_params = { - .cipher_mode = TEST_ECB, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-docsis-128", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-docsis8-128", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS8, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-docsis-256", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-docsis8-256", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS8, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "des-docsis", - .values.job_params = { - .cipher_mode = TEST_DESDOCSIS, - .aes_key_size = 8 - } - }, - { - .name = "des-docsis4", - .values.job_params = { - .cipher_mode = TEST_DESDOCSIS4, - .aes_key_size = 8 - } - }, - { - .name = "des-cbc", - .values.job_params = { - .cipher_mode = TEST_DES, - .aes_key_size = 8 - } - }, - { - .name = "3des-cbc", - .values.job_params = { - .cipher_mode = TEST_3DES, - .aes_key_size = 8 - } - }, - { - .name = "kasumi-uea1", - .values.job_params = { - .cipher_mode = TEST_KASUMI_UEA1, - .aes_key_size = 16 - } - }, - { - .name = "aes-cbcs-1-9", - .values.job_params = { - .cipher_mode = TEST_CBCS_1_9, - .aes_key_size = 16 - } - }, - { - .name = "chacha20", - .values.job_params = { - .cipher_mode = TEST_CHACHA20, - .aes_key_size = 32 - } - }, - { - .name = "snow-v", - .values.job_params = { - .cipher_mode = TEST_SNOW_V, - .aes_key_size = 32 - } - }, -#endif - { - .name = "zuc-eea3", - .values.job_params = { - .cipher_mode = TEST_ZUC_EEA3, - .aes_key_size = 16 - } - }, - { - .name = "zuc-eea3-256", - .values.job_params = { - .cipher_mode = TEST_ZUC_EEA3, - .aes_key_size = 32 - } - }, - { - .name = "snow3g-uea2", - .values.job_params = { - .cipher_mode = TEST_SNOW3G_UEA2, - .aes_key_size = 16 - } - }, - { - .name = "null", - .values.job_params = { - .cipher_mode = TEST_NULL_CIPHER, - .aes_key_size = 0 - } - } +#ifndef __aarch64__ + { .name = "aes-cbc-128", + .values.job_params = { .cipher_mode = TEST_CBC, .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-cbc-192", + .values.job_params = { .cipher_mode = TEST_CBC, .aes_key_size = IMB_KEY_192_BYTES } }, + { .name = "aes-cbc-256", + .values.job_params = { .cipher_mode = TEST_CBC, .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-ctr-128", + .values.job_params = { .cipher_mode = TEST_CNTR, .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-ctr-192", + .values.job_params = { .cipher_mode = TEST_CNTR, .aes_key_size = IMB_KEY_192_BYTES } }, + { .name = "aes-ctr-256", + .values.job_params = { .cipher_mode = TEST_CNTR, .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-ctr8-128", + .values.job_params = { .cipher_mode = TEST_CNTR8, .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-ctr8-192", + .values.job_params = { .cipher_mode = TEST_CNTR8, .aes_key_size = IMB_KEY_192_BYTES } }, + { .name = "aes-ctr8-256", + .values.job_params = { .cipher_mode = TEST_CNTR8, .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-ctr-bit-128", + .values.job_params = { .cipher_mode = TEST_CNTR_BITLEN, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-ctr-bit-192", + .values.job_params = { .cipher_mode = TEST_CNTR_BITLEN, + .aes_key_size = IMB_KEY_192_BYTES } }, + { .name = "aes-ctr-bit-256", + .values.job_params = { .cipher_mode = TEST_CNTR_BITLEN, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-ctr-bit4-128", + .values.job_params = { .cipher_mode = TEST_CNTR_BITLEN4, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-ctr-bit4-192", + .values.job_params = { .cipher_mode = TEST_CNTR_BITLEN4, + .aes_key_size = IMB_KEY_192_BYTES } }, + { .name = "aes-ctr-bit4-256", + .values.job_params = { .cipher_mode = TEST_CNTR_BITLEN4, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-ecb-128", + .values.job_params = { .cipher_mode = TEST_ECB, .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-ecb-192", + .values.job_params = { .cipher_mode = TEST_ECB, .aes_key_size = IMB_KEY_192_BYTES } }, + { .name = "aes-ecb-256", + .values.job_params = { .cipher_mode = TEST_ECB, .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-docsis-128", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-docsis8-128", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS8, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-docsis-256", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-docsis8-256", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS8, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "des-docsis", + .values.job_params = { .cipher_mode = TEST_DESDOCSIS, .aes_key_size = 8 } }, + { .name = "des-docsis4", + .values.job_params = { .cipher_mode = TEST_DESDOCSIS4, .aes_key_size = 8 } }, + { .name = "des-cbc", .values.job_params = { .cipher_mode = TEST_DES, .aes_key_size = 8 } }, + { .name = "3des-cbc", + .values.job_params = { .cipher_mode = TEST_3DES, .aes_key_size = 8 } }, +#endif /* __aarch64__ */ + { .name = "zuc-eea3", + .values.job_params = { .cipher_mode = TEST_ZUC_EEA3, .aes_key_size = 16 } }, + { .name = "zuc-eea3-256", + .values.job_params = { .cipher_mode = TEST_ZUC_EEA3, .aes_key_size = 32 } }, + { .name = "snow3g-uea2", + .values.job_params = { .cipher_mode = TEST_SNOW3G_UEA2, .aes_key_size = 16 } }, +#ifndef __aarch64__ + { .name = "kasumi-uea1", + .values.job_params = { .cipher_mode = TEST_KASUMI_UEA1, .aes_key_size = 16 } }, + { .name = "aes-cbcs-1-9", + .values.job_params = { .cipher_mode = TEST_CBCS_1_9, .aes_key_size = 16 } }, + { .name = "chacha20", + .values.job_params = { .cipher_mode = TEST_CHACHA20, .aes_key_size = 32 } }, + { .name = "snow-v", + .values.job_params = { .cipher_mode = TEST_SNOW_V, .aes_key_size = 32 } }, + { .name = "null", + .values.job_params = { .cipher_mode = TEST_NULL_CIPHER, .aes_key_size = 0 } } +#endif /* __aarch64__ */ }; const struct str_value_mapping hash_algo_str_map[] = { -#ifdef __x86_64__ +#ifndef __aarch64__ { .name = "sha1-hmac", .values.job_params = { @@ -581,6 +427,20 @@ const struct str_value_mapping hash_algo_str_map[] = { .hash_alg = TEST_HASH_CMAC_BITLEN } }, +#endif /* __aarch64__ */ + { + .name = "zuc-eia3", + .values.job_params = { + .hash_alg = TEST_ZUC_EIA3, + } + }, + { + .name = "snow3g-uia2", + .values.job_params = { + .hash_alg = TEST_SNOW3G_UIA2, + } + }, +#ifndef __aarch64__ { .name = "kasumi-uia1", .values.job_params = { @@ -617,6 +477,14 @@ const struct str_value_mapping hash_algo_str_map[] = { .hash_alg = TEST_HASH_POLY1305, } }, +#endif /* __aarch64__ */ + { + .name = "zuc-eia3-256", + .values.job_params = { + .hash_alg = TEST_ZUC256_EIA3, + } + }, +#ifndef __aarch64__ { .name = "crc32-ethernet-fcs", .values.job_params = { @@ -695,145 +563,69 @@ const struct str_value_mapping hash_algo_str_map[] = { .hash_alg = TEST_AUTH_GHASH, } }, -#endif - { - .name = "snow3g-uia2", - .values.job_params = { - .hash_alg = TEST_SNOW3G_UIA2, - } - }, - { - .name = "zuc-eia3", - .values.job_params = { - .hash_alg = TEST_ZUC_EIA3, - } - }, - { - .name = "zuc-eia3-256", - .values.job_params = { - .hash_alg = TEST_ZUC256_EIA3, - } - }, - { - .name = "null", - .values.job_params = { - .hash_alg = TEST_NULL_HASH - } - }, +#endif /* __aarch64__ */ }; const struct str_value_mapping aead_algo_str_map[] = { -#ifdef __x86_64__ - { - .name = "aes-gcm-128", - .values.job_params = { - .cipher_mode = TEST_GCM, - .hash_alg = TEST_HASH_GCM, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-gcm-192", - .values.job_params = { - .cipher_mode = TEST_GCM, - .hash_alg = TEST_HASH_GCM, - .aes_key_size = IMB_KEY_192_BYTES - } - }, - { - .name = "aes-gcm-256", - .values.job_params = { - .cipher_mode = TEST_GCM, - .hash_alg = TEST_HASH_GCM, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-ccm-128", - .values.job_params = { - .cipher_mode = TEST_CCM, - .hash_alg = TEST_HASH_CCM, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-ccm-256", - .values.job_params = { - .cipher_mode = TEST_CCM, - .hash_alg = TEST_HASH_CCM, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "pon-128", - .values.job_params = { - .cipher_mode = TEST_PON_CNTR, - .hash_alg = TEST_PON_CRC_BIP, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "pon-128-no-ctr", - .values.job_params = { - .cipher_mode = TEST_PON_NO_CNTR, - .hash_alg = TEST_PON_CRC_BIP, - .aes_key_size = 0 - } - }, - { - .name = "chacha20-poly1305", - .values.job_params = { - .cipher_mode = TEST_AEAD_CHACHA20, - .hash_alg = TEST_AEAD_POLY1305, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-docsis-128-crc32", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS, - .hash_alg = TEST_DOCSIS_CRC32, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-docsis8-128-crc32", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS8, - .hash_alg = TEST_DOCSIS_CRC32, - .aes_key_size = IMB_KEY_128_BYTES - } - }, - { - .name = "aes-docsis-256-crc32", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS, - .hash_alg = TEST_DOCSIS_CRC32, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "aes-docsis8-256-crc32", - .values.job_params = { - .cipher_mode = TEST_AESDOCSIS8, - .hash_alg = TEST_DOCSIS_CRC32, - .aes_key_size = IMB_KEY_256_BYTES - } - }, - { - .name = "snow-v-aead", - .values.job_params = { - .cipher_mode = TEST_SNOW_V_AEAD, - .aes_key_size = 32, - .hash_alg = TEST_AUTH_SNOW_V_AEAD - } - }, -#endif +#ifndef __aarch64__ + { .name = "aes-gcm-128", + .values.job_params = { .cipher_mode = TEST_GCM, + .hash_alg = TEST_HASH_GCM, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-gcm-192", + .values.job_params = { .cipher_mode = TEST_GCM, + .hash_alg = TEST_HASH_GCM, + .aes_key_size = IMB_KEY_192_BYTES } }, + { .name = "aes-gcm-256", + .values.job_params = { .cipher_mode = TEST_GCM, + .hash_alg = TEST_HASH_GCM, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-ccm-128", + .values.job_params = { .cipher_mode = TEST_CCM, + .hash_alg = TEST_HASH_CCM, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-ccm-256", + .values.job_params = { .cipher_mode = TEST_CCM, + .hash_alg = TEST_HASH_CCM, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "pon-128", + .values.job_params = { .cipher_mode = TEST_PON_CNTR, + .hash_alg = TEST_PON_CRC_BIP, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "pon-128-no-ctr", + .values.job_params = { .cipher_mode = TEST_PON_NO_CNTR, + .hash_alg = TEST_PON_CRC_BIP, + .aes_key_size = 0 } }, + { .name = "chacha20-poly1305", + .values.job_params = { .cipher_mode = TEST_AEAD_CHACHA20, + .hash_alg = TEST_AEAD_POLY1305, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-docsis-128-crc32", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS, + .hash_alg = TEST_DOCSIS_CRC32, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-docsis8-128-crc32", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS8, + .hash_alg = TEST_DOCSIS_CRC32, + .aes_key_size = IMB_KEY_128_BYTES } }, + { .name = "aes-docsis-256-crc32", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS, + .hash_alg = TEST_DOCSIS_CRC32, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "aes-docsis8-256-crc32", + .values.job_params = { .cipher_mode = TEST_AESDOCSIS8, + .hash_alg = TEST_DOCSIS_CRC32, + .aes_key_size = IMB_KEY_256_BYTES } }, + { .name = "snow-v-aead", + .values.job_params = { .cipher_mode = TEST_SNOW_V_AEAD, + .aes_key_size = 32, + .hash_alg = TEST_AUTH_SNOW_V_AEAD } }, +#endif /* __aarch64__ */ }; const struct str_value_mapping cipher_dir_str_map[] = { - {.name = "encrypt", .values.job_params.cipher_dir = IMB_DIR_ENCRYPT}, - {.name = "decrypt", .values.job_params.cipher_dir = IMB_DIR_DECRYPT} + { .name = "encrypt", .values.job_params.cipher_dir = IMB_DIR_ENCRYPT }, + { .name = "decrypt", .values.job_params.cipher_dir = IMB_DIR_DECRYPT } }; /* This struct stores all information about performed test case */ @@ -850,76 +642,68 @@ struct thread_info { IMB_MGR *p_mgr; } t_info[MAX_NUM_THREADS]; -enum cache_type_e { - WARM = 0, - COLD = 1 -}; +enum cache_type_e { WARM = 0, COLD = 1 }; enum cache_type_e cache_type = WARM; const uint32_t auth_tag_length_bytes[] = { - 12, /* SHA1_HMAC */ - 14, /* SHA_224_HMAC */ - 16, /* SHA_256_HMAC */ - 24, /* SHA_384_HMAC */ - 32, /* SHA_512_HMAC */ - 12, /* AES_XCBC */ - 12, /* MD5 */ - 0, /* NULL_HASH */ - 16, /* AES_GMAC */ - 0, /* CUSTOM HASH */ - 16, /* AES_CCM */ - 16, /* AES_CMAC */ - 20, /* PLAIN_SHA1 */ - 28, /* PLAIN_SHA_224 */ - 32, /* PLAIN_SHA_256 */ - 48, /* PLAIN_SHA_384 */ - 64, /* PLAIN_SHA_512 */ - 4, /* AES_CMAC_BITLEN (3GPP) */ - 8, /* PON */ - 4, /* ZUC-EIA3 */ - IMB_DOCSIS_CRC32_TAG_SIZE, /* DOCSIS_CRC32 */ - 4, /* SNOW3G-UIA2 */ - 4, /* KASUMI-UIA1 */ - 16, /* IMB_AUTH_AES_GMAC_128 */ - 16, /* IMB_AUTH_AES_GMAC_192 */ - 16, /* IMB_AUTH_AES_GMAC_256 */ - 16, /* AES_CMAC_256 */ - 16, /* POLY1305 */ - 16, /* AEAD CHACHA20-POLY1305 */ - 16, /* AEAD CHACHA20 with SGL support*/ - 4, /* ZUC-256-EIA3 */ - 16, /* SNOW-V AEAD */ - 16, /* AES-GCM with SGL support */ - 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ - 4, /* IMB_AUTH_CRC32_SCTP */ - 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ - 4, /* IMB_AUTH_CRC24_LTE_A */ - 4, /* IMB_AUTH_CRC24_LTE_B */ - 4, /* IMB_AUTH_CRC16_X25 */ - 4, /* IMB_AUTH_CRC16_FP_DATA */ - 4, /* IMB_AUTH_CRC11_FP_HEADER */ - 4, /* IMB_AUTH_CRC10_IUUP_DATA */ - 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ - 4, /* IMB_AUTH_CRC7_FP_HEADER */ - 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ - 16, /* IMB_AUTH_GHASH */ + 12, /* SHA1_HMAC */ + 14, /* SHA_224_HMAC */ + 16, /* SHA_256_HMAC */ + 24, /* SHA_384_HMAC */ + 32, /* SHA_512_HMAC */ + 12, /* AES_XCBC */ + 12, /* MD5 */ + 0, /* NULL_HASH */ + 16, /* AES_GMAC */ + 0, /* CUSTOM HASH */ + 16, /* AES_CCM */ + 16, /* AES_CMAC */ + 20, /* PLAIN_SHA1 */ + 28, /* PLAIN_SHA_224 */ + 32, /* PLAIN_SHA_256 */ + 48, /* PLAIN_SHA_384 */ + 64, /* PLAIN_SHA_512 */ + 4, /* AES_CMAC_BITLEN (3GPP) */ + 8, /* PON */ + 4, /* ZUC-EIA3 */ + IMB_DOCSIS_CRC32_TAG_SIZE, /* DOCSIS_CRC32 */ + 4, /* SNOW3G-UIA2 */ + 4, /* KASUMI-UIA1 */ + 16, /* IMB_AUTH_AES_GMAC_128 */ + 16, /* IMB_AUTH_AES_GMAC_192 */ + 16, /* IMB_AUTH_AES_GMAC_256 */ + 16, /* AES_CMAC_256 */ + 16, /* POLY1305 */ + 16, /* AEAD CHACHA20-POLY1305 */ + 16, /* AEAD CHACHA20 with SGL support*/ + 4, /* ZUC-256-EIA3 */ + 16, /* SNOW-V AEAD */ + 16, /* AES-GCM with SGL support */ + 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ + 4, /* IMB_AUTH_CRC32_SCTP */ + 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ + 4, /* IMB_AUTH_CRC24_LTE_A */ + 4, /* IMB_AUTH_CRC24_LTE_B */ + 4, /* IMB_AUTH_CRC16_X25 */ + 4, /* IMB_AUTH_CRC16_FP_DATA */ + 4, /* IMB_AUTH_CRC11_FP_HEADER */ + 4, /* IMB_AUTH_CRC10_IUUP_DATA */ + 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ + 4, /* IMB_AUTH_CRC7_FP_HEADER */ + 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ }; uint32_t index_limit; uint32_t key_idxs[NUM_OFFSETS]; uint32_t offsets[NUM_OFFSETS]; -uint32_t sha_size_incr = 24; +uint32_t sha_size_incr = UINT32_MAX; +uint32_t buffer_offset = 0; -enum range { - RANGE_MIN = 0, - RANGE_STEP, - RANGE_MAX, - NUM_RANGE -}; +enum range { RANGE_MIN = 0, RANGE_STEP, RANGE_MAX, NUM_RANGE }; -uint32_t job_sizes[NUM_RANGE] = {DEFAULT_JOB_SIZE_MIN, - DEFAULT_JOB_SIZE_STEP, - DEFAULT_JOB_SIZE_MAX}; +uint32_t job_sizes[NUM_RANGE] = { DEFAULT_JOB_SIZE_MIN, DEFAULT_JOB_SIZE_STEP, + DEFAULT_JOB_SIZE_MAX }; uint32_t job_size_list[MAX_LIST]; uint32_t job_size_count = 0; uint32_t imix_list[MAX_LIST]; @@ -942,19 +726,21 @@ uint64_t ccm_aad_size = DEFAULT_CCM_AAD_SIZE; uint64_t chacha_poly_aad_size = DEFAULT_CHACHA_POLY_AAD_SIZE; uint64_t snow_v_aad_size = DEFAULT_SNOW_V_AEAD_AAD_SIZE; -struct custom_job_params custom_job_params = { - .cipher_mode = TEST_NULL_CIPHER, - .hash_alg = TEST_NULL_HASH, - .aes_key_size = 0, - .cipher_dir = IMB_DIR_ENCRYPT -}; +struct custom_job_params custom_job_params = { .cipher_mode = TEST_NULL_CIPHER, + .hash_alg = TEST_NULL_HASH, + .aes_key_size = 0, + .cipher_dir = IMB_DIR_ENCRYPT }; -uint8_t archs[NUM_ARCHS] = {1, 1, 1, 1, 1, 1}; /* uses all function sets */ +#ifndef __aarch64__ +uint8_t archs[NUM_ARCHS] = { 1, 1, 1, 1 }; /* uses all function sets */ +#else +uint8_t archs[NUM_ARCHS] = { 1, 1 }; /* uses all function sets */ +#endif int use_job_api = 0; int use_gcm_sgl_api = 0; int use_unhalted_cycles = 0; /* read unhalted cycles instead of tsc */ uint64_t rd_cycles_cost = 0; /* cost of reading unhalted cycles */ -uint64_t core_mask = 0; /* bitmap of selected cores */ +uint64_t core_mask = 0; /* bitmap of selected cores */ uint64_t flags = 0; /* flags passed to alloc_mb_mgr() */ @@ -971,8 +757,10 @@ static uint32_t pb_mod = 0; static int silent_progress_bar = 0; static int plot_output_option = 0; +static int quic_api_test = 0; + /* API types */ -typedef enum { +typedef enum { TEST_API_JOB = 0, TEST_API_BURST, TEST_API_CIPHER_BURST, @@ -980,19 +768,19 @@ typedef enum { TEST_API_NUMOF } TEST_API; -const char *str_api_list[TEST_API_NUMOF] = {"single job", "burst", - "cipher-only burst", - "hash-only burst"}; +const char *str_api_list[TEST_API_NUMOF] = { "single job", "burst", "cipher-only burst", + "hash-only burst" }; static TEST_API test_api = TEST_API_JOB; /* test job API by default */ -static uint32_t burst_size = 0; /* num jobs to pass to burst API */ -static uint32_t segment_size = 0; /* segment size to test SGL (0 = no SGL) */ +static uint32_t burst_size = 0; /* num jobs to pass to burst API */ +static uint32_t segment_size = 0; /* segment size to test SGL (0 = no SGL) */ static volatile int timebox_on = 1; /* flag to stop the test loop */ static int use_timebox = 1; /* time-box feature on/off flag */ #ifdef LINUX -static void timebox_callback(int sig) +static void +timebox_callback(int sig) { (void) sig; timebox_on = 0; @@ -1000,7 +788,8 @@ static void timebox_callback(int sig) #endif #ifdef _WIN32 -static void CALLBACK timebox_callback(PVOID lpParam, BOOLEAN TimerFired) +static void CALLBACK +timebox_callback(PVOID lpParam, BOOLEAN TimerFired) { (void) lpParam; (void) TimerFired; @@ -1009,25 +798,26 @@ static void CALLBACK timebox_callback(PVOID lpParam, BOOLEAN TimerFired) #endif /* Return rdtsc to core cycle scale factor */ -static double get_tsc_to_core_scale(const int turbo) +static double +get_tsc_to_core_scale(const int turbo) { - int i, num_loops = 1; + int i; /* use enough cycles for accurate measurement */ const uint64_t expected_cycles = 1000000000; - uint64_t tsc_cycles; + uint64_t tsc_cycles = 0; /* if turbo enabled then run longer */ /* to allow frequency to stabilize */ - if (turbo) - num_loops = 8; + const int num_loops = turbo ? 8 : 1; for (i = 0; i < num_loops; i++) tsc_cycles = measure_tsc(expected_cycles); - return ((double)tsc_cycles / (double)expected_cycles); + return ((double) tsc_cycles / (double) expected_cycles); } -static void prog_bar_init(const uint32_t total_num) +static void +prog_bar_init(const uint32_t total_num) { if (silent_progress_bar) return; @@ -1049,7 +839,8 @@ static void prog_bar_init(const uint32_t total_num) fputs(prog_bar, stderr); } -static void prog_bar_fini(void) +static void +prog_bar_fini(void) { if (silent_progress_bar) return; @@ -1058,7 +849,8 @@ static void prog_bar_fini(void) fputs(prog_bar, stderr); } -static void prog_bar_update(const uint32_t num) +static void +prog_bar_update(const uint32_t num) { if (silent_progress_bar) return; @@ -1072,7 +864,7 @@ static void prog_bar_update(const uint32_t num) if (pb_idx < (PB_SIZE + 1)) pb_idx++; } else { - const char pb_inter_chars[] = {'|', '/', '-', '\\'}; + const char pb_inter_chars[] = { '|', '/', '-', '\\' }; /* print intermediate chars */ prog_bar[pb_idx] = pb_inter_chars[num % DIM(pb_inter_chars)]; fputs(prog_bar, stderr); @@ -1080,14 +872,16 @@ static void prog_bar_update(const uint32_t num) } /* Read unhalted cycles */ -__forceinline uint64_t read_cycles(const uint32_t core) +__forceinline uint64_t +read_cycles(const uint32_t core) { uint64_t val = 0; - if (msr_read(core, IA32_MSR_CPU_UNHALTED_THREAD, - &val) != MACHINE_RETVAL_OK) { - fprintf(stderr, "Error reading cycles " - "counter on core %u!\n", core); + if (msr_read(core, IA32_MSR_CPU_UNHALTED_THREAD, &val) != MACHINE_RETVAL_OK) { + fprintf(stderr, + "Error reading cycles " + "counter on core %u!\n", + core); exit(EXIT_FAILURE); } @@ -1095,13 +889,15 @@ __forceinline uint64_t read_cycles(const uint32_t core) } /* Method used by qsort to compare 2 values */ -static int compare_uint64_t(const void *a, const void *b) +static int +compare_uint64_t(const void *a, const void *b) { - return (int)(int64_t)(*(const uint64_t *)a - *(const uint64_t *)b); + return (int) (int64_t) (*(const uint64_t *) a - *(const uint64_t *) b); } /* Get number of bits set in value */ -static unsigned bitcount(const uint64_t val) +static unsigned +bitcount(const uint64_t val) { unsigned i, bits = 0; @@ -1114,8 +910,8 @@ static unsigned bitcount(const uint64_t val) /* Get the next core in core mask Set last_core to negative to start from beginning of core_mask */ -static int next_core(const uint64_t core_mask, - const int last_core) +static int +next_core(const uint64_t core_mask, const int last_core) { int core = 0; @@ -1125,7 +921,7 @@ static int next_core(const uint64_t core_mask, while (((core_mask >> core) & 1) == 0) { core++; - if (core >= (int)BITS(core_mask)) + if (core >= (int) BITS(core_mask)) return -1; } @@ -1133,7 +929,8 @@ static int next_core(const uint64_t core_mask, } /* Set CPU affinity for current thread */ -static int set_affinity(const int cpu) +static int +set_affinity(const int cpu) { int ret = 0; int num_cpus = 0; @@ -1154,15 +951,16 @@ static int set_affinity(const int cpu) /* Check if selected core is valid */ if (cpu < 0 || cpu >= num_cpus) { - fprintf(stderr, "Invalid CPU selected! " - "Max valid CPU is %d\n", num_cpus - 1); + fprintf(stderr, + "Invalid CPU selected! " + "Max valid CPU is %d\n", + num_cpus - 1); return 1; } #ifdef _WIN32 NewGroupAffinity.Mask = 1ULL << cpu; - ret = !SetThreadGroupAffinity(GetCurrentThread(), - &NewGroupAffinity, NULL); + ret = !SetThreadGroupAffinity(GetCurrentThread(), &NewGroupAffinity, NULL); #else cpu_set_t cpuset; @@ -1171,8 +969,7 @@ static int set_affinity(const int cpu) /* Set affinity of current process to cpu */ #if defined(__FreeBSD__) - ret = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, - sizeof(cpuset), &cpuset); + ret = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset); #else ret = sched_setaffinity(0, sizeof(cpuset), &cpuset); #endif @@ -1182,7 +979,8 @@ static int set_affinity(const int cpu) } /* Start counting unhalted cycles */ -static int start_cycles_ctr(const uint32_t core) +static int +start_cycles_ctr(const uint32_t core) { int ret; @@ -1205,11 +1003,12 @@ static int start_cycles_ctr(const uint32_t core) return ret; /* Enable cycles counter */ - return msr_write(core, IA32_MSR_PERF_GLOBAL_CTR, (1ULL << 33)); + return msr_write(core, IA32_MSR_PERF_GLOBAL_CTR, (1ULL << 33)); } /* Init MSR module */ -static int init_msr_mod(void) +static int +init_msr_mod(void) { unsigned max_core_count = 0; #ifdef _WIN32 @@ -1226,7 +1025,8 @@ static int init_msr_mod(void) } /* Set the cost of reading unhalted cycles using RDMSR */ -static int set_unhalted_cycle_cost(const int core, uint64_t *value) +static int +set_unhalted_cycle_cost(const int core, uint64_t *value) { uint64_t time1, time2; @@ -1243,7 +1043,8 @@ static int set_unhalted_cycle_cost(const int core, uint64_t *value) } /* Calculate the general cost of reading unhalted cycles (median) */ -static int set_avg_unhalted_cycle_cost(const int core, uint64_t *value) +static int +set_avg_unhalted_cycle_cost(const int core, uint64_t *value) { unsigned i; uint64_t cycles[10]; @@ -1260,23 +1061,14 @@ static int set_avg_unhalted_cycle_cost(const int core, uint64_t *value) qsort(cycles, DIM(cycles), sizeof(uint64_t), compare_uint64_t); /* set median cost */ - *value = cycles[DIM(cycles)/2]; + *value = cycles[DIM(cycles) / 2]; return 0; } -static inline uint64_t perf_rdtscp(void) -{ -#ifdef __aarch64__ - return rdtscp(); -#else - uint32_t aux; - return __rdtscp(&aux); -#endif -} - /* Freeing allocated memory */ -static void free_mem(uint8_t **p_buffer, imb_uint128_t **p_keys) +static void +free_mem(uint8_t **p_buffer, imb_uint128_t **p_keys) { imb_uint128_t *keys = NULL; uint8_t *buf = NULL; @@ -1312,24 +1104,28 @@ get_key_pointer(const uint32_t index, const imb_uint128_t *p_keys) return (const void *) &p_keys[key_idxs[index]]; } -static uint8_t *get_src_buffer(const uint32_t index, uint8_t *p_buffer) +static uint8_t * +get_src_buffer(const uint32_t index, uint8_t *p_buffer) { - return &p_buffer[offsets[index]]; + return &p_buffer[offsets[index] + buffer_offset]; } -static uint8_t *get_dst_buffer(const uint32_t index, uint8_t *p_buffer) +static uint8_t * +get_dst_buffer(const uint32_t index, uint8_t *p_buffer) { - return &p_buffer[offsets[index] + sha_size_incr]; + return &p_buffer[offsets[index] + buffer_offset + sha_size_incr]; } -static uint32_t get_next_index(uint32_t index) +static uint32_t +get_next_index(uint32_t index) { if (++index >= index_limit) index = 0; return index; } -static void init_buf(void *pb, const size_t length) +static void +init_buf(void *pb, const size_t length) { const size_t n = length / sizeof(uint64_t); size_t i = 0; @@ -1338,7 +1134,7 @@ static void init_buf(void *pb, const size_t length) return; for (i = 0; i < n; i++) - ((uint64_t *)pb)[i] = (uint64_t) rand(); + ((uint64_t *) pb)[i] = (uint64_t) rand(); } /* @@ -1346,11 +1142,11 @@ static void init_buf(void *pb, const size_t length) * init_offsets() needs to be called prior to that so that * index_limit is set up accordingly to hot/cold selection. */ -static void init_mem(uint8_t **p_buffer, imb_uint128_t **p_keys) +static void +init_mem(uint8_t **p_buffer, imb_uint128_t **p_keys) { const size_t bufs_size = index_limit * REGION_SIZE; - const size_t keys_size = - index_limit * KEYS_PER_JOB * sizeof(imb_uint128_t); + const size_t keys_size = index_limit * KEYS_PER_JOB * sizeof(imb_uint128_t); const size_t alignment = 64; uint8_t *buf = NULL; imb_uint128_t *keys = NULL; @@ -1405,7 +1201,8 @@ static void init_mem(uint8_t **p_buffer, imb_uint128_t **p_keys) * Initialize packet buffer and keys offsets from * the start of the respective buffers */ -static void init_offsets(const enum cache_type_e ctype) +static void +init_offsets(const enum cache_type_e ctype) { if (ctype == COLD) { uint32_t i; @@ -1444,8 +1241,7 @@ static void init_offsets(const enum cache_type_e ctype) const uint32_t L1_way_size = 4096; key_idxs[i] = i * KEYS_PER_JOB; - offsets[i] = i * REGION_SIZE + - ((i * offset_step) & (L1_way_size - 1)); + offsets[i] = i * REGION_SIZE + ((i * offset_step) & (L1_way_size - 1)); } } } @@ -1547,8 +1343,8 @@ get_next_size(const uint32_t index) } static inline void -set_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, - const uint32_t i, const uint32_t index) +set_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, const uint32_t i, + const uint32_t index, const IMB_JOB *template) { uint32_t list_idx; @@ -1559,13 +1355,15 @@ set_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, job->msg_len_to_cipher_in_bytes = cipher_size_list[list_idx]; job->msg_len_to_hash_in_bytes = hash_size_list[list_idx]; + } else { + job->msg_len_to_hash_in_bytes = template->msg_len_to_hash_in_bytes; + job->msg_len_to_cipher_in_bytes = template->msg_len_to_cipher_in_bytes; } if (job->hash_alg == IMB_AUTH_PON_CRC_BIP) { - uint64_t *p_src = - (uint64_t *) get_src_buffer(index, p_buffer); + uint64_t *p_src = (uint64_t *) get_src_buffer(index, p_buffer); - job->src = (const uint8_t *)p_src; + job->src = (const uint8_t *) p_src; if (imix_list_count != 0) p_src[0] = xgem_hdr_list[list_idx]; else @@ -1578,41 +1376,30 @@ set_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, job->u.GCM.aad = job->src; } else if (job->cipher_mode == IMB_CIPHER_CCM) { job->u.CCM.aad = job->src; - job->enc_keys = job->dec_keys = - (const uint32_t *) get_key_pointer(index, - p_keys); + job->enc_keys = job->dec_keys = (const uint32_t *) get_key_pointer(index, p_keys); } else if (job->cipher_mode == IMB_CIPHER_DES3) { static const void *ks_ptr[3]; - ks_ptr[0] = ks_ptr[1] = ks_ptr[2] = - get_key_pointer(index, p_keys); - job->enc_keys = - job->dec_keys = ks_ptr; + ks_ptr[0] = ks_ptr[1] = ks_ptr[2] = get_key_pointer(index, p_keys); + job->enc_keys = job->dec_keys = ks_ptr; } else if (job->cipher_mode == IMB_CIPHER_CHACHA20_POLY1305) { job->u.CHACHA20_POLY1305.aad = job->src; } else if (job->cipher_mode == IMB_CIPHER_SNOW_V_AEAD) { job->u.SNOW_V_AEAD.aad = job->src; - job->enc_keys = job->dec_keys = - (const uint32_t *) get_key_pointer(index, - p_keys); - /* Force destination buffer to start 8 bytes after source */ + job->enc_keys = job->dec_keys = (const uint32_t *) get_key_pointer(index, p_keys); + /* Force destination buffer to start 8 bytes after source */ } else if (job->cipher_mode == IMB_CIPHER_PON_AES_CNTR) { job->dst = get_src_buffer(index, p_buffer) + 8; - job->enc_keys = job->dec_keys = - (const uint32_t *) get_key_pointer(index, - p_keys); + job->enc_keys = job->dec_keys = (const uint32_t *) get_key_pointer(index, p_keys); } else { - job->enc_keys = job->dec_keys = - (const uint32_t *) get_key_pointer(index, - p_keys); + job->enc_keys = job->dec_keys = (const uint32_t *) get_key_pointer(index, p_keys); } } static inline void -set_sgl_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, - const uint32_t size_idx, const uint32_t buf_index, - struct IMB_SGL_IOV *sgl, struct gcm_context_data *gcm_ctx, - struct chacha20_poly1305_context_data *cp_ctx) +set_sgl_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, const uint32_t size_idx, + const uint32_t buf_index, struct IMB_SGL_IOV *sgl, + struct gcm_context_data *gcm_ctx, struct chacha20_poly1305_context_data *cp_ctx) { uint8_t *src = get_src_buffer(buf_index, p_buffer); uint8_t *dst = get_dst_buffer(buf_index, p_buffer); @@ -1625,10 +1412,11 @@ set_sgl_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, /* If IMIX testing is being done, set the buffer size to cipher and hash * going through the list of sizes precalculated */ if (imix_list_count != 0) { - uint32_t list_idx = size_idx & (JOB_SIZE_IMIX_LIST - 1); + const uint32_t list_idx = size_idx & (JOB_SIZE_IMIX_LIST - 1); job->msg_len_to_cipher_in_bytes = cipher_size_list[list_idx]; } + buf_size = (uint32_t) job->msg_len_to_cipher_in_bytes; if (job->cipher_mode == IMB_CIPHER_GCM_SGL) { job->u.GCM.aad = aad; @@ -1637,9 +1425,7 @@ set_sgl_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, job->u.CHACHA20_POLY1305.aad = aad; job->u.CHACHA20_POLY1305.ctx = cp_ctx; } - job->enc_keys = job->dec_keys = - (const uint32_t *) get_key_pointer(buf_index, - p_keys); + job->enc_keys = job->dec_keys = (const uint32_t *) get_key_pointer(buf_index, p_keys); job->sgl_state = IMB_SGL_ALL; const uint32_t num_segs = buf_size / segment_size; @@ -1663,8 +1449,8 @@ set_sgl_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, }; static void -set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, - uint64_t *xgem_hdr_list, struct params_s *params) +set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, uint64_t *xgem_hdr_list, + struct params_s *params) { unsigned int i, list_size; uint32_t job_size; @@ -1680,8 +1466,7 @@ set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, else job_size = params->size_aes; - if ((params->cipher_mode == TEST_AESDOCSIS8) || - (params->cipher_mode == TEST_CNTR8)) + if ((params->cipher_mode == TEST_AESDOCSIS8) || (params->cipher_mode == TEST_CNTR8)) cipher_size_list[i] = job_size + 8; else if (params->cipher_mode == TEST_DESDOCSIS4) cipher_size_list[i] = job_size + 4; @@ -1698,13 +1483,11 @@ set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, if (job_size < 8) cipher_size_list[i] = 8; else - cipher_size_list[i] = - (job_size + 3) & 0xfffffffc; + cipher_size_list[i] = (job_size + 3) & 0xfffffffc; } else cipher_size_list[i] = job_size; - if ((params->hash_alg == TEST_HASH_CCM) || - (params->hash_alg == TEST_HASH_GCM)) + if ((params->hash_alg == TEST_HASH_CCM) || (params->hash_alg == TEST_HASH_GCM)) hash_size_list[i] = job_size; else hash_size_list[i] = job_size + sha_size_incr; @@ -1731,11 +1514,11 @@ set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, hash_size_list[i] = 0; if (((params->cipher_mode == TEST_AESDOCSIS) || - (params->cipher_mode == TEST_AESDOCSIS8)) && + (params->cipher_mode == TEST_AESDOCSIS8)) && (params->hash_alg == TEST_DOCSIS_CRC32)) { const uint32_t ciph_adjust = /* SA + DA */ - IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE - 2; - /* ETH TYPE */ + IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE - 2; + /* ETH TYPE */ hash_size_list[i] = cipher_size_list[i] + ciph_adjust; cipher_size_list[i] -= IMB_DOCSIS_CRC32_TAG_SIZE; @@ -1743,20 +1526,194 @@ set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, if (params->hash_alg == TEST_PON_CRC_BIP) { /* create XGEM header template */ - const uint64_t pli = - (job_size << 2) & 0xffff; + const uint64_t pli = (job_size << 2) & 0xffff; + + xgem_hdr_list[i] = ((pli >> 8) & 0xff) | ((pli & 0xff) << 8); + } + } +} + +/* + * ========================================================= + * QUIC specific benchmarks + * ========================================================= + */ + +#define MAX_K 32 +#define PKT_SIZE_MAX 16384 + +#define AAD_LEN 12 +#define IV_LEN 12 +#define TAG_LEN 16 + +#ifndef __aarch64__ +static void +bench_quic_aes_gcm(IMB_MGR *p_mgr, const uint64_t n_max, uint8_t *blob) +{ + void *out[MAX_K]; + const void *in[MAX_K]; + const void *iv[MAX_K]; + const void *aad[MAX_K]; + void *tag[MAX_K]; + uint64_t len[MAX_K]; + const uint8_t kp[16] = { 0xaa, 055, 0x11, 0x44, 0x01, 0x02, 0x00, 0x03, + 0xaa, 055, 0x11, 0x44, 0x01, 0x02, 0x00, 0x03 }; + struct gcm_key_data key; + const uint64_t pkt_size = 16384; + const int K = MAX_K; + uint32_t aux; + uint64_t n; + int i; + + printf("QUIC-API AES-GCM-128 test start...\n" + " packet-size: %lu\n" + " number of packets : %d\n" + " iterations: %lu\n", + (unsigned long) pkt_size, K, (unsigned long) n_max); + + for (n = 0, i = 0; i < K; i++) { + iv[i] = &blob[n]; + n += IV_LEN; + + aad[i] = &blob[n]; + n += AAD_LEN; + + in[i] = &blob[n]; + n += pkt_size; + + out[i] = &blob[n]; + n += pkt_size; + + tag[i] = &blob[n]; + n += TAG_LEN; + + len[i] = pkt_size; + } + + IMB_AES128_GCM_PRE(p_mgr, kp, &key); + + const uint64_t tsc_start = __rdtscp(&aux); + + for (n = 0; n < n_max; n++) { + imb_quic_aes_gcm(p_mgr, &key, IMB_KEY_128_BYTES, IMB_DIR_ENCRYPT, (void **) out, + (const void *const *) in, len, (const void *const *) iv, + (const void *const *) &aad, AAD_LEN, (void **) tag, TAG_LEN, K); + } + + const uint64_t tsc_end = __rdtscp(&aux); + const uint64_t cpi = (tsc_end - tsc_start) / n_max; + const uint64_t cpp = cpi / (uint64_t) K; - xgem_hdr_list[i] = ((pli >> 8) & 0xff) | - ((pli & 0xff) << 8); + printf("QUIC-API AES-GCM-128 cycles/iteration = %lu, " + "cycles/packet = %lu\n", + (unsigned long) cpi, (unsigned long) cpp); +} + +static void +bench_quic_aes_ecb_hp(IMB_MGR *p_mgr, const uint64_t n_max, uint8_t *blob) +{ + void *out[MAX_K]; + const void *in[MAX_K]; + const uint8_t kp[16] = { 0xaa, 055, 0x11, 0x44, 0x01, 0x02, 0x00, 0x03, + 0xaa, 055, 0x11, 0x44, 0x01, 0x02, 0x00, 0x03 }; + DECLARE_ALIGNED(uint32_t enc_keys[15 * 4], 16); + DECLARE_ALIGNED(uint32_t dec_keys[15 * 4], 16); + const int K = MAX_K; + const uint64_t pkt_size = 16; + uint32_t aux; + uint64_t n; + int i; + + for (n = 0, i = 0; i < K; i++) { + in[i] = &blob[n]; + n += pkt_size; + + out[i] = &blob[n]; + n += pkt_size; + } + + printf("QUIC-API AES-ECB-128 test start...\n" + " packet-size: %lu\n" + " number of packets : %d\n" + " iterations: %lu\n", + (unsigned long) pkt_size, K, (unsigned long) n_max); + + IMB_AES_KEYEXP_128(p_mgr, kp, enc_keys, dec_keys); + + const uint64_t tsc_start = __rdtscp(&aux); + + for (n = 0; n < n_max; n++) + imb_quic_hp_aes_ecb(p_mgr, enc_keys, (void **) out, (const void *const *) in, K, + IMB_KEY_128_BYTES); + + const uint64_t tsc_end = __rdtscp(&aux); + const uint64_t cpi = (tsc_end - tsc_start) / n_max; + const uint64_t cpp = cpi / (uint64_t) K; + + printf("QUIC-API AES-ECB-128 cycles/iteration = %lu, " + "cycles/packet = %lu\n", + (unsigned long) cpi, (unsigned long) cpp); +} + +static void +quic_main(IMB_MGR *p_mgr) +{ + const size_t blob_sz = (AAD_LEN + IV_LEN + TAG_LEN + PKT_SIZE_MAX) * MAX_K * 2; + + uint8_t *blob_ptr = (uint8_t *) malloc(blob_sz); + + if (blob_ptr == NULL) { + printf("Error allocating QUIC-API test buffer!\n"); + exit(EXIT_FAILURE); + } + + memset(blob_ptr, 0xaa, blob_sz); + + enum arch_type_e arch; + + for (arch = ARCH_SSE; arch <= ARCH_AVX512; arch++) { + if (archs[arch] == 0) + continue; + + switch (arch) { + case ARCH_SSE: + printf("SSE\n"); + init_mb_mgr_sse(p_mgr); + break; + case ARCH_AVX: + printf("AVX\n"); + init_mb_mgr_avx(p_mgr); + break; + case ARCH_AVX2: + printf("AVX2\n"); + init_mb_mgr_avx2(p_mgr); + break; + default: /* ARCH_AV512 */ + printf("AVX512\n"); + init_mb_mgr_avx512(p_mgr); + break; + } + + if (imb_get_errno(p_mgr) != 0) { + printf("Error initializing MB_MGR! %s\n", + imb_get_strerror(imb_get_errno(p_mgr))); + free(blob_ptr); + exit(EXIT_FAILURE); } + const uint64_t n_max = 50000; + + bench_quic_aes_ecb_hp(p_mgr, n_max, blob_ptr); + bench_quic_aes_gcm(p_mgr, n_max, blob_ptr); } + free(blob_ptr); } +#endif /* __aarch64__ */ /* Performs test using AES_HMAC or DOCSIS */ static uint64_t -do_test(IMB_MGR *mb_mgr, struct params_s *params, - const uint32_t num_iter, uint8_t *p_buffer, imb_uint128_t *p_keys) +do_test(IMB_MGR *mb_mgr, struct params_s *params, const uint32_t num_iter, uint8_t *p_buffer, + imb_uint128_t *p_keys) { IMB_JOB *job; IMB_JOB job_template; @@ -1770,12 +1727,12 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, static DECLARE_ALIGNED(uint8_t k3[16], 16); static DECLARE_ALIGNED(struct gcm_key_data gdata_key, 512); uint64_t time = 0; + uint32_t aux; uint8_t gcm_key[32]; uint8_t next_iv[IMB_AES_BLOCK_SIZE]; - IMB_JOB jobs[MAX_BURST_SIZE]; struct gcm_context_data gcm_ctx[MAX_BURST_SIZE]; struct chacha20_poly1305_context_data cp_ctx[MAX_BURST_SIZE]; - struct IMB_SGL_IOV *sgl[MAX_BURST_SIZE] = {NULL}; + struct IMB_SGL_IOV *sgl[MAX_BURST_SIZE] = { NULL }; uint32_t max_num_segs = 1; memset(&job_template, 0, sizeof(IMB_JOB)); @@ -1785,12 +1742,10 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, set_size_lists(cipher_size_list, hash_size_list, xgem_hdr_list, params); if (segment_size != 0) - max_num_segs = DIV_ROUND_UP(job_sizes[RANGE_MAX], - segment_size); + max_num_segs = DIV_ROUND_UP(job_sizes[RANGE_MAX], segment_size); for (i = 0; i < MAX_BURST_SIZE; i++) { - sgl[i] = malloc(sizeof(struct IMB_SGL_IOV) * - max_num_segs); + sgl[i] = malloc(sizeof(struct IMB_SGL_IOV) * max_num_segs); if (sgl[i] == NULL) { fprintf(stderr, "malloc() failed\n"); goto exit; @@ -1894,7 +1849,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, case TEST_SNOW3G_UIA2: job_template.hash_alg = IMB_AUTH_SNOW3G_UIA2_BITLEN; job_template.u.SNOW3G_UIA2._key = k3; - job_template.u.SNOW3G_UIA2._iv = (uint8_t *)&auth_iv; + job_template.u.SNOW3G_UIA2._iv = (uint8_t *) &auth_iv; break; case TEST_KASUMI_UIA1: job_template.hash_alg = IMB_AUTH_KASUMI_UIA1; @@ -1968,16 +1923,14 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, break; default: /* HMAC hash alg is SHA1 or MD5 */ - job_template.u.HMAC._hashed_auth_key_xor_ipad = - (uint8_t *) ipad; - job_template.u.HMAC._hashed_auth_key_xor_opad = - (uint8_t *) opad; + job_template.u.HMAC._hashed_auth_key_xor_ipad = (uint8_t *) ipad; + job_template.u.HMAC._hashed_auth_key_xor_opad = (uint8_t *) opad; job_template.hash_alg = (IMB_HASH_ALG) params->hash_alg; break; } if (tag_size == 0) job_template.auth_tag_output_len_in_bytes = - (uint64_t) auth_tag_length_bytes[job_template.hash_alg - 1]; + (uint64_t) auth_tag_length_bytes[job_template.hash_alg - 1]; else job_template.auth_tag_output_len_in_bytes = tag_size; @@ -1985,10 +1938,9 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, if (params->cipher_mode == TEST_NULL_CIPHER) { job_template.chain_order = IMB_ORDER_HASH_CIPHER; - } else if (params->cipher_mode == TEST_CCM || - ((params->cipher_mode == TEST_AESDOCSIS || - params->cipher_mode == TEST_AESDOCSIS8) && - params->hash_alg == TEST_DOCSIS_CRC32)) { + } else if (params->cipher_mode == TEST_CCM || ((params->cipher_mode == TEST_AESDOCSIS || + params->cipher_mode == TEST_AESDOCSIS8) && + params->hash_alg == TEST_DOCSIS_CRC32)) { if (job_template.cipher_direction == IMB_DIR_ENCRYPT) job_template.chain_order = IMB_ORDER_HASH_CIPHER; else @@ -2069,17 +2021,15 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job_template.cipher_start_src_offset_in_bytes = 0; job_template.enc_keys = k1_expanded; job_template.dec_keys = k1_expanded; - job_template.u.CHACHA20_POLY1305.aad_len_in_bytes = - params->aad_size; + job_template.u.CHACHA20_POLY1305.aad_len_in_bytes = params->aad_size; job_template.iv_len_in_bytes = 12; } else if (job_template.cipher_mode == IMB_CIPHER_SNOW_V) job_template.iv_len_in_bytes = 16; else if (job_template.cipher_mode == IMB_CIPHER_SNOW_V_AEAD && - job_template.hash_alg == IMB_AUTH_SNOW_V_AEAD) { + job_template.hash_alg == IMB_AUTH_SNOW_V_AEAD) { job_template.key_len_in_bytes = 32; job_template.iv_len_in_bytes = 16; - job_template.u.SNOW_V_AEAD.aad_len_in_bytes = - params->aad_size; + job_template.u.SNOW_V_AEAD.aad_len_in_bytes = params->aad_size; } #define TIMEOUT_MS 100 /*< max time for one packet size to be tested for */ @@ -2107,16 +2057,14 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, /* create the timer queue */ hTimeboxQueue = CreateTimerQueue(); if (NULL == hTimeboxQueue) { - fprintf(stderr, "CreateTimerQueue() error %u\n", - (unsigned) GetLastError()); + fprintf(stderr, "CreateTimerQueue() error %u\n", (unsigned) GetLastError()); goto exit; } /* set a timer to call the timebox */ if (!CreateTimerQueueTimer(&hTimebox, hTimeboxQueue, - (WAITORTIMERCALLBACK) - timebox_callback, - NULL, TIMEOUT_MS, 0, 0)) { + (WAITORTIMERCALLBACK) timebox_callback, NULL, TIMEOUT_MS, + 0, 0)) { fprintf(stderr, "CreateTimerQueueTimer() error %u\n", (unsigned) GetLastError()); goto exit; @@ -2130,67 +2078,80 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, time = read_cycles(params->core); else #endif - time = perf_rdtscp(); + time = __rdtscp(&aux); /* test burst api */ if (test_api == TEST_API_BURST) { uint32_t num_jobs = num_iter; - IMB_JOB *jobs[IMB_MAX_BURST_SIZE] = {NULL}; +#ifdef DEBUG + uint32_t jobs_submitted = 0; +#endif + + IMB_JOB *jobs[IMB_MAX_BURST_SIZE] = { NULL }; + + imb_set_session(mb_mgr, &job_template); while (num_jobs && timebox_on) { - uint32_t n = (num_jobs / burst_size) ? - burst_size : num_jobs; + uint32_t n = (num_jobs / burst_size) ? burst_size : num_jobs; while (IMB_GET_NEXT_BURST(mb_mgr, n, jobs) < n) IMB_FLUSH_BURST(mb_mgr, n, jobs); /* set all job params */ for (i = 0; i < n; i++) { - IMB_JOB *job = jobs[i]; - *job = job_template; + job = jobs[i]; + + if (job->session_id != job_template.session_id) + *job = job_template; if (segment_size != 0) - set_sgl_job_fields(job, p_buffer, - p_keys, i, - index, sgl[i], - &gcm_ctx[i], - &cp_ctx[i]); + set_sgl_job_fields(job, p_buffer, p_keys, i, index, sgl[i], + &gcm_ctx[i], &cp_ctx[i]); else - set_job_fields(job, p_buffer, p_keys, - i, index); + set_job_fields(job, p_buffer, p_keys, i, index, + &job_template); index = get_next_index(index); - } /* submit burst */ #ifdef DEBUG - jobs_done += IMB_SUBMIT_BURST(mb_mgr, n, jobs); - if (jobs_done == 0) { + const uint32_t ret = IMB_SUBMIT_BURST(mb_mgr, n, jobs); + + if (ret == 0) { const int err = imb_get_errno(mb_mgr); if (err != 0) { - printf("submit_burst error %d : '%s'\n", - err, imb_get_strerror(err)); + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); } } + jobs_done += ret; + jobs_submitted += n; #else - jobs_done += - IMB_SUBMIT_BURST_NOCHECK(mb_mgr, n, jobs); + jobs_done += IMB_SUBMIT_BURST_NOCHECK(mb_mgr, n, jobs); #endif num_jobs -= n; } - jobs_done += - IMB_FLUSH_BURST(mb_mgr, IMB_MAX_BURST_SIZE, jobs); + jobs_done += IMB_FLUSH_BURST(mb_mgr, IMB_MAX_BURST_SIZE, jobs); + +#ifdef DEBUG + if (jobs_done != jobs_submitted) { + printf("Number of jobs completed (%u) not equal to " + "jobs submitted (%u)\n", + jobs_done, jobs_submitted); + goto exit; + } +#endif /* test cipher-only burst api */ } else if (test_api == TEST_API_CIPHER_BURST) { + IMB_JOB jobs[MAX_BURST_SIZE]; IMB_JOB *jt = &job_template; uint32_t num_jobs = num_iter; uint32_t list_idx; while (num_jobs && timebox_on) { - uint32_t n_jobs = - (num_jobs / burst_size) ? burst_size : num_jobs; + uint32_t n_jobs = (num_jobs / burst_size) ? burst_size : num_jobs; /* set all job params */ for (i = 0; i < n_jobs; i++) { @@ -2210,8 +2171,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job->src = get_src_buffer(index, p_buffer); job->dst = get_dst_buffer(index, p_buffer); job->enc_keys = job->dec_keys = - (const uint32_t *) - get_key_pointer(index, p_keys); + (const uint32_t *) get_key_pointer(index, p_keys); job->cipher_start_src_offset_in_bytes = jt->cipher_start_src_offset_in_bytes; job->iv = jt->iv; @@ -2222,25 +2182,21 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, /* submit cipher-only burst */ #ifdef DEBUG const uint32_t completed_jobs = - IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, n_jobs, - jt->cipher_mode, - jt->cipher_direction, - jt->key_len_in_bytes); + IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, n_jobs, jt->cipher_mode, + jt->cipher_direction, jt->key_len_in_bytes); if (completed_jobs != n_jobs) { const int err = imb_get_errno(mb_mgr); if (err != 0) { printf("submit_cipher_burst error " - "%d : '%s'\n", err, - imb_get_strerror(err)); + "%d : '%s'\n", + err, imb_get_strerror(err)); } } #else - IMB_SUBMIT_CIPHER_BURST_NOCHECK(mb_mgr, jobs, n_jobs, - jt->cipher_mode, - jt->cipher_direction, - jt->key_len_in_bytes); + IMB_SUBMIT_CIPHER_BURST_NOCHECK(mb_mgr, jobs, n_jobs, jt->cipher_mode, + jt->cipher_direction, jt->key_len_in_bytes); #endif num_jobs -= n_jobs; } @@ -2248,13 +2204,13 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, /* test hash-only burst api */ } else if (test_api == TEST_API_HASH_BURST) { + IMB_JOB jobs[MAX_BURST_SIZE]; IMB_JOB *jt = &job_template; uint32_t num_jobs = num_iter; uint32_t list_idx; while (num_jobs && timebox_on) { - uint32_t n_jobs = - (num_jobs / burst_size) ? burst_size : num_jobs; + uint32_t n_jobs = (num_jobs / burst_size) ? burst_size : num_jobs; /* set all job params */ for (i = 0; i < n_jobs; i++) { @@ -2265,8 +2221,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, * list of sizes precalculated */ if (imix_list_count != 0) { list_idx = i & (JOB_SIZE_IMIX_LIST - 1); - job->msg_len_to_hash_in_bytes = - hash_size_list[list_idx]; + job->msg_len_to_hash_in_bytes = hash_size_list[list_idx]; } else job->msg_len_to_hash_in_bytes = jt->msg_len_to_hash_in_bytes; @@ -2287,38 +2242,38 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, /* submit hash-only burst */ #ifdef DEBUG const uint32_t completed_jobs = - IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, n_jobs, - jt->hash_alg); + IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, n_jobs, jt->hash_alg); if (completed_jobs != n_jobs) { const int err = imb_get_errno(mb_mgr); if (err != 0) { printf("submit_hash_burst error " - "%d : '%s'\n", err, - imb_get_strerror(err)); + "%d : '%s'\n", + err, imb_get_strerror(err)); } } #else - IMB_SUBMIT_HASH_BURST_NOCHECK(mb_mgr, jobs, n_jobs, - jt->hash_alg); + IMB_SUBMIT_HASH_BURST_NOCHECK(mb_mgr, jobs, n_jobs, jt->hash_alg); #endif num_jobs -= n_jobs; } jobs_done = num_iter - num_jobs; - } else { /* test job api */ + } else { /* TEST_API_JOB */ + imb_set_session(mb_mgr, &job_template); + for (i = 0; (i < num_iter) && timebox_on; i++) { job = IMB_GET_NEXT_JOB(mb_mgr); - *job = job_template; + + if (job->session_id != job_template.session_id) + *job = job_template; if (segment_size != 0) - set_sgl_job_fields(job, p_buffer, p_keys, - i, index, - sgl[0], &gcm_ctx[0], - &cp_ctx[0]); + set_sgl_job_fields(job, p_buffer, p_keys, i, index, sgl[0], + &gcm_ctx[0], &cp_ctx[0]); else - set_job_fields(job, p_buffer, p_keys, i, index); + set_job_fields(job, p_buffer, p_keys, i, index, &job_template); index = get_next_index(index); #ifdef DEBUG @@ -2329,9 +2284,10 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, while (job) { #ifdef DEBUG if (job->status != IMB_STATUS_COMPLETED) { - fprintf(stderr, - "failed job, status:%d\n", - job->status); + const int err = imb_get_errno(mb_mgr); + + fprintf(stderr, "failed job, status:%d, %s\n", job->status, + imb_get_strerror(err)); goto exit; } #endif @@ -2345,14 +2301,12 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, if (job->status != IMB_STATUS_COMPLETED) { const int errc = imb_get_errno(mb_mgr); - fprintf(stderr, - "failed job, status:%d, " - "error code:%d, %s\n", job->status, - errc, imb_get_strerror(errc)); + fprintf(stderr, "failed job, status:%d, error:%d, %s\n", + job->status, errc, imb_get_strerror(errc)); goto exit; } #else - (void)job; + (void) job; #endif } @@ -2368,7 +2322,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, time = (read_cycles(params->core) - rd_cycles_cost) - time; else #endif - time = perf_rdtscp() - time; + time = __rdtscp(&aux) - time; if (use_timebox) { #ifdef LINUX @@ -2384,8 +2338,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #else /* _WIN32 */ /* delete all timeboxes in the timer queue */ if (!DeleteTimerQueue(hTimeboxQueue)) - fprintf(stderr, "DeleteTimerQueue() error %u\n", - (unsigned) GetLastError()); + fprintf(stderr, "DeleteTimerQueue() error %u\n", (unsigned) GetLastError()); #endif /* calculate return value */ @@ -2407,14 +2360,12 @@ exit: exit(EXIT_FAILURE); } +#ifndef __aarch64__ static void run_gcm_sgl(aes_gcm_init_t init, aes_gcm_enc_dec_update_t update, - aes_gcm_enc_dec_finalize_t finalize, - struct gcm_key_data *gdata_key, - struct gcm_context_data *gdata_ctx, - uint8_t *p_buffer, uint32_t buf_size, - const void *aad, const uint64_t aad_size, - const uint32_t num_iter) + aes_gcm_enc_dec_finalize_t finalize, struct gcm_key_data *gdata_key, + struct gcm_context_data *gdata_ctx, uint8_t *p_buffer, uint32_t buf_size, + const void *aad, const uint64_t aad_size, const uint32_t num_iter) { uint32_t i; static uint32_t index = 0; @@ -2435,17 +2386,12 @@ run_gcm_sgl(aes_gcm_init_t init, aes_gcm_enc_dec_update_t update, init(gdata_key, gdata_ctx, iv, aad, aad_size); for (j = 0; j < num_segs; j++) - update(gdata_key, gdata_ctx, - &pb[j*segment_size], - &pb[j*segment_size], - segment_size); + update(gdata_key, gdata_ctx, &pb[j * segment_size], + &pb[j * segment_size], segment_size); if (final_seg_sz != 0) - update(gdata_key, gdata_ctx, - &pb[j*segment_size], - &pb[j*segment_size], - final_seg_sz); - finalize(gdata_key, gdata_ctx, auth_tag, - sizeof(auth_tag)); + update(gdata_key, gdata_ctx, &pb[j * segment_size], + &pb[j * segment_size], final_seg_sz); + finalize(gdata_key, gdata_ctx, auth_tag, sizeof(auth_tag)); index = get_next_index(index); } @@ -2458,8 +2404,7 @@ run_gcm_sgl(aes_gcm_init_t init, aes_gcm_enc_dec_update_t update, init(gdata_key, gdata_ctx, iv, aad, aad_size); update(gdata_key, gdata_ctx, pb, pb, buf_size); - finalize(gdata_key, gdata_ctx, auth_tag, - sizeof(auth_tag)); + finalize(gdata_key, gdata_ctx, auth_tag, sizeof(auth_tag)); index = get_next_index(index); } @@ -2467,12 +2412,9 @@ run_gcm_sgl(aes_gcm_init_t init, aes_gcm_enc_dec_update_t update, } static void -run_gcm(aes_gcm_enc_dec_t enc_dec, - struct gcm_key_data *gdata_key, - struct gcm_context_data *gdata_ctx, - uint8_t *p_buffer, uint32_t buf_size, - const void *aad, const uint64_t aad_size, - const uint32_t num_iter) +run_gcm(aes_gcm_enc_dec_t enc_dec, struct gcm_key_data *gdata_key, + struct gcm_context_data *gdata_ctx, uint8_t *p_buffer, uint32_t buf_size, const void *aad, + const uint64_t aad_size, const uint32_t num_iter) { uint32_t i; uint32_t index = 0; @@ -2485,9 +2427,8 @@ run_gcm(aes_gcm_enc_dec_t enc_dec, if (imix_list_count != 0) buf_size = get_next_size(i); - enc_dec(gdata_key, gdata_ctx, pb, pb, - buf_size, iv, aad, aad_size, - auth_tag, sizeof(auth_tag)); + enc_dec(gdata_key, gdata_ctx, pb, pb, buf_size, iv, aad, aad_size, auth_tag, + sizeof(auth_tag)); index = get_next_index(index); } @@ -2495,15 +2436,15 @@ run_gcm(aes_gcm_enc_dec_t enc_dec, /* Performs test using GCM */ static uint64_t -do_test_gcm(struct params_s *params, - const uint32_t num_iter, IMB_MGR *mb_mgr, - uint8_t *p_buffer, imb_uint128_t *p_keys) +do_test_gcm(struct params_s *params, const uint32_t num_iter, IMB_MGR *mb_mgr, uint8_t *p_buffer, + imb_uint128_t *p_keys) { static DECLARE_ALIGNED(struct gcm_key_data gdata_key, 512); static DECLARE_ALIGNED(struct gcm_context_data gdata_ctx, 64); uint8_t *key; uint8_t *aad = NULL; uint64_t time = 0; + uint32_t aux; /* Force SGL API if segment size is not 0 */ if (segment_size != 0) @@ -2544,122 +2485,84 @@ do_test_gcm(struct params_s *params, time = read_cycles(params->core); else #endif - time = perf_rdtscp(); + time = __rdtscp(&aux); if (params->aes_key_size == IMB_KEY_128_BYTES) { if (use_gcm_sgl_api) - run_gcm_sgl(mb_mgr->gcm128_init, - mb_mgr->gcm128_enc_update, - mb_mgr->gcm128_enc_finalize, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, + run_gcm_sgl(mb_mgr->gcm128_init, mb_mgr->gcm128_enc_update, + mb_mgr->gcm128_enc_finalize, &gdata_key, &gdata_ctx, + p_buffer, params->size_aes, aad, params->aad_size, num_iter); else - run_gcm(mb_mgr->gcm128_enc, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, - num_iter); + run_gcm(mb_mgr->gcm128_enc, &gdata_key, &gdata_ctx, p_buffer, + params->size_aes, aad, params->aad_size, num_iter); } else if (params->aes_key_size == IMB_KEY_192_BYTES) { if (use_gcm_sgl_api) - run_gcm_sgl(mb_mgr->gcm192_init, - mb_mgr->gcm192_enc_update, - mb_mgr->gcm192_enc_finalize, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, + run_gcm_sgl(mb_mgr->gcm192_init, mb_mgr->gcm192_enc_update, + mb_mgr->gcm192_enc_finalize, &gdata_key, &gdata_ctx, + p_buffer, params->size_aes, aad, params->aad_size, num_iter); else - run_gcm(mb_mgr->gcm192_enc, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, - num_iter); + run_gcm(mb_mgr->gcm192_enc, &gdata_key, &gdata_ctx, p_buffer, + params->size_aes, aad, params->aad_size, num_iter); } else { /* 256 */ if (use_gcm_sgl_api) - run_gcm_sgl(mb_mgr->gcm256_init, - mb_mgr->gcm256_enc_update, - mb_mgr->gcm256_enc_finalize, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, + run_gcm_sgl(mb_mgr->gcm256_init, mb_mgr->gcm256_enc_update, + mb_mgr->gcm256_enc_finalize, &gdata_key, &gdata_ctx, + p_buffer, params->size_aes, aad, params->aad_size, num_iter); else - run_gcm(mb_mgr->gcm256_enc, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, - num_iter); + run_gcm(mb_mgr->gcm256_enc, &gdata_key, &gdata_ctx, p_buffer, + params->size_aes, aad, params->aad_size, num_iter); } #ifndef _WIN32 if (use_unhalted_cycles) - time = (read_cycles(params->core) - - rd_cycles_cost) - time; + time = (read_cycles(params->core) - rd_cycles_cost) - time; else #endif - time = perf_rdtscp() - time; + time = __rdtscp(&aux) - time; } else { /*DECRYPT*/ #ifndef _WIN32 if (use_unhalted_cycles) time = read_cycles(params->core); else #endif - time = perf_rdtscp(); + time = __rdtscp(&aux); if (params->aes_key_size == IMB_KEY_128_BYTES) { if (use_gcm_sgl_api) - run_gcm_sgl(mb_mgr->gcm128_init, - mb_mgr->gcm128_dec_update, - mb_mgr->gcm128_dec_finalize, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, + run_gcm_sgl(mb_mgr->gcm128_init, mb_mgr->gcm128_dec_update, + mb_mgr->gcm128_dec_finalize, &gdata_key, &gdata_ctx, + p_buffer, params->size_aes, aad, params->aad_size, num_iter); else - run_gcm(mb_mgr->gcm128_dec, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, - num_iter); + run_gcm(mb_mgr->gcm128_dec, &gdata_key, &gdata_ctx, p_buffer, + params->size_aes, aad, params->aad_size, num_iter); } else if (params->aes_key_size == IMB_KEY_192_BYTES) { if (use_gcm_sgl_api) - run_gcm_sgl(mb_mgr->gcm192_init, - mb_mgr->gcm192_dec_update, - mb_mgr->gcm192_dec_finalize, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, + run_gcm_sgl(mb_mgr->gcm192_init, mb_mgr->gcm192_dec_update, + mb_mgr->gcm192_dec_finalize, &gdata_key, &gdata_ctx, + p_buffer, params->size_aes, aad, params->aad_size, num_iter); else - run_gcm(mb_mgr->gcm192_dec, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, - num_iter); + run_gcm(mb_mgr->gcm192_dec, &gdata_key, &gdata_ctx, p_buffer, + params->size_aes, aad, params->aad_size, num_iter); } else { /* 256 */ if (use_gcm_sgl_api) - run_gcm_sgl(mb_mgr->gcm256_init, - mb_mgr->gcm256_dec_update, - mb_mgr->gcm256_dec_finalize, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, + run_gcm_sgl(mb_mgr->gcm256_init, mb_mgr->gcm256_dec_update, + mb_mgr->gcm256_dec_finalize, &gdata_key, &gdata_ctx, + p_buffer, params->size_aes, aad, params->aad_size, num_iter); else - run_gcm(mb_mgr->gcm256_dec, - &gdata_key, &gdata_ctx, - p_buffer, params->size_aes, - aad, params->aad_size, - num_iter); + run_gcm(mb_mgr->gcm256_dec, &gdata_key, &gdata_ctx, p_buffer, + params->size_aes, aad, params->aad_size, num_iter); } #ifndef _WIN32 if (use_unhalted_cycles) - time = (read_cycles(params->core) - - rd_cycles_cost) - time; + time = (read_cycles(params->core) - rd_cycles_cost) - time; else #endif - time = perf_rdtscp() - time; + time = __rdtscp(&aux) - time; } free(key); @@ -2673,8 +2576,7 @@ do_test_gcm(struct params_s *params, /* Performs test using CHACHA20-POLY1305 direct API */ static uint64_t -do_test_chacha_poly(struct params_s *params, - const uint32_t num_iter, IMB_MGR *mb_mgr, +do_test_chacha_poly(struct params_s *params, const uint32_t num_iter, IMB_MGR *mb_mgr, uint8_t *p_buffer, imb_uint128_t *p_keys) { uint8_t key[32]; @@ -2682,9 +2584,7 @@ do_test_chacha_poly(struct params_s *params, DECLARE_ALIGNED(uint8_t iv[16], 16); uint8_t *aad = NULL; uint64_t time = 0; -#ifdef __x86_64__ uint32_t aux; -#endif struct chacha20_poly1305_context_data chacha_ctx; static uint32_t index = 0; uint32_t num_segs; @@ -2711,12 +2611,8 @@ do_test_chacha_poly(struct params_s *params, time = read_cycles(params->core); else #endif - -#ifdef __aarch64__ - time = rdtscp(); -#else time = __rdtscp(&aux); -#endif + for (i = 0; i < num_iter; i++) { uint8_t *pb = get_dst_buffer(index, p_buffer); @@ -2732,57 +2628,40 @@ do_test_chacha_poly(struct params_s *params, } } - IMB_CHACHA20_POLY1305_INIT(mb_mgr, key, &chacha_ctx, iv, - aad, params->aad_size); + IMB_CHACHA20_POLY1305_INIT(mb_mgr, key, &chacha_ctx, iv, aad, params->aad_size); if (params->cipher_dir == IMB_DIR_ENCRYPT) { for (j = 0; j < num_segs; j++) - IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, - &chacha_ctx, - &pb[j*segment_size], - &pb[j*segment_size], - segment_size); + IMB_CHACHA20_POLY1305_ENC_UPDATE( + mb_mgr, key, &chacha_ctx, &pb[j * segment_size], + &pb[j * segment_size], segment_size); if (final_seg_sz != 0) - IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, - &chacha_ctx, - &pb[j*segment_size], - &pb[j*segment_size], - final_seg_sz); - IMB_CHACHA20_POLY1305_ENC_FINALIZE(mb_mgr, - &chacha_ctx, - auth_tag, + IMB_CHACHA20_POLY1305_ENC_UPDATE( + mb_mgr, key, &chacha_ctx, &pb[j * segment_size], + &pb[j * segment_size], final_seg_sz); + IMB_CHACHA20_POLY1305_ENC_FINALIZE(mb_mgr, &chacha_ctx, auth_tag, sizeof(auth_tag)); } else { /* IMB_DIR_DECRYPT */ for (j = 0; j < num_segs; j++) - IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, - &chacha_ctx, - &pb[j*segment_size], - &pb[j*segment_size], - segment_size); + IMB_CHACHA20_POLY1305_ENC_UPDATE( + mb_mgr, key, &chacha_ctx, &pb[j * segment_size], + &pb[j * segment_size], segment_size); if (final_seg_sz != 0) - IMB_CHACHA20_POLY1305_DEC_UPDATE(mb_mgr, key, - &chacha_ctx, - &pb[j*segment_size], - &pb[j*segment_size], - final_seg_sz); - IMB_CHACHA20_POLY1305_DEC_FINALIZE(mb_mgr, - &chacha_ctx, - auth_tag, + IMB_CHACHA20_POLY1305_DEC_UPDATE( + mb_mgr, key, &chacha_ctx, &pb[j * segment_size], + &pb[j * segment_size], final_seg_sz); + IMB_CHACHA20_POLY1305_DEC_FINALIZE(mb_mgr, &chacha_ctx, auth_tag, sizeof(auth_tag)); } index = get_next_index(index); } #ifndef _WIN32 if (use_unhalted_cycles) - time = (read_cycles(params->core) - - rd_cycles_cost) - time; + time = (read_cycles(params->core) - rd_cycles_cost) - time; else #endif -#ifdef __aarch64__ - time = rdtscp() - time; -#else time = __rdtscp(&aux) - time; -#endif + free(aad); if (!num_iter) @@ -2793,15 +2672,12 @@ do_test_chacha_poly(struct params_s *params, /* Performs test using GCM */ static uint64_t -do_test_ghash(struct params_s *params, - const uint32_t num_iter, IMB_MGR *mb_mgr, - uint8_t *p_buffer, imb_uint128_t *p_keys) +do_test_ghash(struct params_s *params, const uint32_t num_iter, IMB_MGR *mb_mgr, uint8_t *p_buffer, + imb_uint128_t *p_keys) { static DECLARE_ALIGNED(struct gcm_key_data gdata_key, 512); uint64_t time = 0; -#ifdef __x86_64__ uint32_t aux; -#endif uint32_t i, index = 0; uint8_t auth_tag[16]; @@ -2812,18 +2688,14 @@ do_test_ghash(struct params_s *params, time = read_cycles(params->core); else #endif -#ifdef __aarch64__ - time = rdtscp(); -#else time = __rdtscp(&aux); -#endif + if (imix_list_count != 0) { for (i = 0; i < num_iter; i++) { uint8_t *pb = get_dst_buffer(index, p_buffer); const uint32_t buf_size = get_next_size(i); - IMB_GHASH(mb_mgr, &gdata_key, pb, buf_size, - auth_tag, sizeof(auth_tag)); + IMB_GHASH(mb_mgr, &gdata_key, pb, buf_size, auth_tag, sizeof(auth_tag)); index = get_next_index(index); } } else { @@ -2831,33 +2703,28 @@ do_test_ghash(struct params_s *params, uint8_t *pb = get_dst_buffer(index, p_buffer); const uint32_t buf_size = params->size_aes; - IMB_GHASH(mb_mgr, &gdata_key, pb, buf_size, - auth_tag, sizeof(auth_tag)); + IMB_GHASH(mb_mgr, &gdata_key, pb, buf_size, auth_tag, sizeof(auth_tag)); index = get_next_index(index); } } #ifndef _WIN32 if (use_unhalted_cycles) - time = (read_cycles(params->core) - - rd_cycles_cost) - time; + time = (read_cycles(params->core) - rd_cycles_cost) - time; else #endif -#ifdef __aarch64__ - time = rdtscp() - time; -#else time = __rdtscp(&aux) - time; -#endif + if (!num_iter) return time; return time / num_iter; } +#endif /* __aarch64__ */ /* Computes mean of set of times after dropping bottom and top quarters */ static uint64_t -mean_median(uint64_t *array, uint32_t size, - uint8_t *p_buffer, imb_uint128_t *p_keys) +mean_median(uint64_t *array, uint32_t size, uint8_t *p_buffer, imb_uint128_t *p_keys) { const uint32_t quarter = size / 4; uint32_t i; @@ -2877,7 +2744,6 @@ mean_median(uint64_t *array, uint32_t size, array += quarter; size -= quarter * 2; - if ((size == 0) || (size & 0x80000000)) { fprintf(stderr, "Not enough data points!\n"); free_mem(&p_buffer, &p_keys); @@ -2893,10 +2759,9 @@ mean_median(uint64_t *array, uint32_t size, /* Runs test for each buffer size and stores averaged execution time */ static void -process_variant(IMB_MGR *mgr, const enum arch_type_e arch, - struct params_s *params, - struct variant_s *variant_ptr, const uint32_t run, - uint8_t *p_buffer, imb_uint128_t *p_keys) +process_variant(IMB_MGR *mgr, const enum arch_type_e arch, struct params_s *params, + struct variant_s *variant_ptr, const uint32_t run, uint8_t *p_buffer, + imb_uint128_t *p_keys) { uint32_t sizes = params->num_sizes; uint64_t *times = &variant_ptr->avg_times[run]; @@ -2908,8 +2773,7 @@ process_variant(IMB_MGR *mgr, const enum arch_type_e arch, for (sz = 0; sz < sizes; sz++) { if (job_size_count == 0) - size_aes = job_sizes[RANGE_MIN] + - (sz * job_sizes[RANGE_STEP]); + size_aes = job_sizes[RANGE_MIN] + (sz * job_sizes[RANGE_STEP]); else size_aes = job_size_list[sz]; @@ -2933,49 +2797,42 @@ process_variant(IMB_MGR *mgr, const enum arch_type_e arch, * (only allowed for GCM/CCM) */ if (size_aes == 0 && params->aad_size != 0) - num_iter = (iter_scale >= (uint32_t)params->aad_size) ? - (iter_scale / (uint32_t)params->aad_size) : - 1; + num_iter = (iter_scale >= (uint32_t) params->aad_size) + ? (iter_scale / (uint32_t) params->aad_size) + : 1; else if (size_aes != 0) - num_iter = (iter_scale >= size_aes) ? - (iter_scale / size_aes) : 1; + num_iter = (iter_scale >= size_aes) ? (iter_scale / size_aes) : 1; else num_iter = iter_scale; params->size_aes = size_aes; +#ifndef __aarch64__ if (params->cipher_mode == TEST_GCM && (!use_job_api)) { if (job_iter == 0) - *times = do_test_gcm(params, 2 * num_iter, mgr, - p_buffer, p_keys); + *times = do_test_gcm(params, 2 * num_iter, mgr, p_buffer, p_keys); else - *times = do_test_gcm(params, job_iter, mgr, - p_buffer, p_keys); - } else if (params->cipher_mode == TEST_AEAD_CHACHA20 && - (!use_job_api)) { + *times = do_test_gcm(params, job_iter, mgr, p_buffer, p_keys); + } else if (params->cipher_mode == TEST_AEAD_CHACHA20 && (!use_job_api)) { if (job_iter == 0) - *times = do_test_chacha_poly(params, - 2 * num_iter, mgr, - p_buffer, p_keys); + *times = do_test_chacha_poly(params, 2 * num_iter, mgr, p_buffer, + p_keys); else - *times = do_test_chacha_poly(params, - job_iter, mgr, - p_buffer, p_keys); - } else if (params->hash_alg == TEST_AUTH_GHASH && - (!use_job_api)) { + *times = do_test_chacha_poly(params, job_iter, mgr, p_buffer, + p_keys); + } else if (params->hash_alg == TEST_AUTH_GHASH && (!use_job_api)) { if (job_iter == 0) - *times = do_test_ghash(params, 2 * num_iter, - mgr, p_buffer, p_keys); + *times = do_test_ghash(params, 2 * num_iter, mgr, p_buffer, p_keys); else - *times = do_test_ghash(params, job_iter, mgr, - p_buffer, p_keys); + *times = do_test_ghash(params, job_iter, mgr, p_buffer, p_keys); } else { +#endif if (job_iter == 0) - *times = do_test(mgr, params, num_iter, - p_buffer, p_keys); + *times = do_test(mgr, params, num_iter, p_buffer, p_keys); else - *times = do_test(mgr, params, job_iter, - p_buffer, p_keys); + *times = do_test(mgr, params, job_iter, p_buffer, p_keys); +#ifndef __aarch64__ } +#endif times += NUM_RUNS; } @@ -2985,9 +2842,8 @@ process_variant(IMB_MGR *mgr, const enum arch_type_e arch, /* Generates output containing averaged times for each test variant */ static void -print_times(struct variant_s *variant_list, struct params_s *params, - const uint32_t total_variants, uint8_t *p_buffer, - imb_uint128_t *p_keys) +print_times(struct variant_s *variant_list, struct params_s *params, const uint32_t total_variants, + uint8_t *p_buffer, imb_uint128_t *p_keys) { /* If IMIX is used, only show the average size */ const uint32_t sizes = (imix_list_count != 0) ? 1 : params->num_sizes; @@ -2995,35 +2851,81 @@ print_times(struct variant_s *variant_list, struct params_s *params, uint32_t sz; if (plot_output_option == 0) { - const char *func_names[NUM_ARCHS] = { - "SSE", "AVX", "AVX2", "AVX512", "AARCH64", "SVE256" - }; - const char *c_mode_names[TEST_NUM_CIPHER_TESTS - 1] = { - "CBC", "CNTR", "CNTR+8", "CNTR_BITLEN", "CNTR_BITLEN4", - "ECB", "CBCS_1_9", "NULL_CIPHER", "DOCAES", "DOCAES+8", - "DOCDES", "DOCDES+4", "GCM", "CCM", "DES", "3DES", - "PON", "PON_NO_CTR", "ZUC_EEA3", "SNOW3G_UEA2_BITLEN", - "KASUMI_UEA1_BITLEN", "CHACHA20", "CHACHA20_AEAD", - "SNOW_V", "SNOW_V_AEAD" - }; - const char *c_dir_names[2] = { - "ENCRYPT", "DECRYPT" - }; - const char *h_alg_names[TEST_NUM_HASH_TESTS - 1] = { - "SHA1_HMAC", "SHA_224_HMAC", "SHA_256_HMAC", - "SHA_384_HMAC", "SHA_512_HMAC", "XCBC", - "MD5", "CMAC", "SHA1", "SHA_224", "SHA_256", - "SHA_384", "SHA_512", "CMAC_BITLEN", "CMAC_256", - "NULL_HASH", "CRC32", "GCM", "CUSTOM", "CCM", - "BIP-CRC32", "ZUC_EIA3_BITLEN", "SNOW3G_UIA2_BITLEN", - "KASUMI_UIA1", "GMAC-128", "GMAC-192", "GMAC-256", - "POLY1305", "POLY1305_AEAD", "ZUC256_EIA3", - "SNOW_V_AEAD", "CRC32_ETH_FCS", "CRC32_SCTP", - "CRC32_WIMAX_DATA", "CRC24_LTE_A", "CR24_LTE_B", - "CR16_X25", "CRC16_FP_DATA", "CRC11_FP_HEADER", - "CRC10_IUUP_DATA", "CRC8_WIMAX_HCS", "CRC7_FP_HEADER", - "CRC6_IUUP_HEADER", "GHASH" - }; +#ifndef __aarch64__ + const char *func_names[4] = { "SSE", "AVX", "AVX2", "AVX512" }; +#else + const char *func_names[2] = { "AARCH64", "SVE256" }; +#endif + const char *c_mode_names[TEST_NUM_CIPHER_TESTS - 1] = { "CBC", + "CNTR", + "CNTR+8", + "CNTR_BITLEN", + "CNTR_BITLEN4", + "ECB", + "CBCS_1_9", + "NULL_CIPHER", + "DOCAES", + "DOCAES+8", + "DOCDES", + "DOCDES+4", + "GCM", + "CCM", + "DES", + "3DES", + "PON", + "PON_NO_CTR", + "ZUC_EEA3", + "SNOW3G_UEA2_BITLEN", + "KASUMI_UEA1_BITLEN", + "CHACHA20", + "CHACHA20_AEAD", + "SNOW_V", + "SNOW_V_AEAD" }; + const char *c_dir_names[2] = { "ENCRYPT", "DECRYPT" }; + const char *h_alg_names[TEST_NUM_HASH_TESTS - 1] = { "SHA1_HMAC", + "SHA_224_HMAC", + "SHA_256_HMAC", + "SHA_384_HMAC", + "SHA_512_HMAC", + "XCBC", + "MD5", + "CMAC", + "SHA1", + "SHA_224", + "SHA_256", + "SHA_384", + "SHA_512", + "CMAC_BITLEN", + "CMAC_256", + "NULL_HASH", + "CRC32", + "GCM", + "CUSTOM", + "CCM", + "BIP-CRC32", + "ZUC_EIA3_BITLEN", + "SNOW3G_UIA2_BITLEN", + "KASUMI_UIA1", + "GMAC-128", + "GMAC-192", + "GMAC-256", + "POLY1305", + "POLY1305_AEAD", + "ZUC256_EIA3", + "SNOW_V_AEAD", + "CRC32_ETH_FCS", + "CRC32_SCTP", + "CRC32_WIMAX_DATA", + "CRC24_LTE_A", + "CR24_LTE_B", + "CR16_X25", + "CRC16_FP_DATA", + "CRC11_FP_HEADER", + "CRC10_IUUP_DATA", + "CRC8_WIMAX_HCS", + "CRC7_FP_HEADER", + "CRC6_IUUP_HEADER", + "GHASH" }; struct params_s par; printf("ARCH"); @@ -3069,16 +2971,13 @@ print_times(struct variant_s *variant_list, struct params_s *params, if (imix_list_count != 0) printf("%u", average_job_size); else if (job_size_count == 0) - printf("%d", job_sizes[RANGE_MIN] + - (sz * job_sizes[RANGE_STEP])); + printf("%d", job_sizes[RANGE_MIN] + (sz * job_sizes[RANGE_STEP])); else printf("%d", job_size_list[sz]); for (col = 0; col < total_variants; col++) { - uint64_t *time_ptr = - &variant_list[col].avg_times[sz * NUM_RUNS]; + uint64_t *time_ptr = &variant_list[col].avg_times[sz * NUM_RUNS]; const unsigned long long val = - mean_median(time_ptr, NUM_RUNS, - p_buffer, p_keys); + mean_median(time_ptr, NUM_RUNS, p_buffer, p_keys); printf("\t%llu", val); } @@ -3095,7 +2994,7 @@ static void * run_tests(void *arg) { uint32_t i; - struct thread_info *info = (struct thread_info *)arg; + struct thread_info *info = (struct thread_info *) arg; IMB_MGR *p_mgr = NULL; struct params_s params; enum arch_type_e arch; @@ -3112,6 +3011,17 @@ run_tests(void *arg) p_mgr = info->p_mgr; +#ifndef __aarch64__ + if (quic_api_test) { + quic_main(p_mgr); +#ifndef _WIN32 + return NULL; +#else + return; +#endif + } +#endif /* __aarch64__ */ + memset(¶ms, 0, sizeof(params)); if (job_size_count == 0) @@ -3119,13 +3029,15 @@ run_tests(void *arg) else params.num_sizes = job_size_count; - params.core = (uint32_t)info->core; + params.core = (uint32_t) info->core; /* if cores selected then set affinity */ if (core_mask) if (set_affinity(info->core) != 0) { - fprintf(stderr, "Failed to set cpu " - "affinity on core %d\n", info->core); + fprintf(stderr, + "Failed to set cpu " + "affinity on core %d\n", + info->core); goto exit_failure; } @@ -3136,27 +3048,34 @@ run_tests(void *arg) ret = start_cycles_ctr(params.core); if (ret != 0) { - fprintf(stderr, "Failed to start cycles " - "counter on core %u\n", params.core); + fprintf(stderr, + "Failed to start cycles " + "counter on core %u\n", + params.core); goto exit_failure; } /* Get average cost of reading counter */ ret = set_avg_unhalted_cycle_cost(params.core, &rd_cycles_cost); if (ret != 0 || rd_cycles_cost == 0) { fprintf(stderr, "Error calculating unhalted " - "cycles read overhead!\n"); + "cycles read overhead!\n"); goto exit_failure; } else - fprintf(stderr, "Started counting unhalted cycles on " + fprintf(stderr, + "Started counting unhalted cycles on " "core %u\nUnhalted cycles read cost = %lu " - "cycles\n", params.core, - (unsigned long)rd_cycles_cost); + "cycles\n", + params.core, (unsigned long) rd_cycles_cost); } init_mem(&buf, &keys); /* Calculating number of all variants */ +#ifndef __aarch64__ for (arch = ARCH_SSE; arch < NUM_ARCHS; arch++) { +#else + for (arch = ARCH_AARCH64; arch < NUM_ARCHS; arch++) { +#endif if (archs[arch] == 0) continue; total_variants++; @@ -3168,12 +3087,12 @@ run_tests(void *arg) } if (info->print_info && !silent_progress_bar) - fprintf(stderr, "Total number of combinations (algos, " + fprintf(stderr, + "Total number of combinations (algos, " "key sizes, cipher directions) to test = %u\n", total_variants); - variant_list = (struct variant_s *) - malloc(total_variants * sizeof(struct variant_s)); + variant_list = (struct variant_s *) malloc(total_variants * sizeof(struct variant_s)); if (variant_list == NULL) { fprintf(stderr, "Cannot allocate memory\n"); goto exit_failure; @@ -3181,8 +3100,7 @@ run_tests(void *arg) memset(variant_list, 0, total_variants * sizeof(struct variant_s)); at_size = NUM_RUNS * params.num_sizes * sizeof(uint64_t); - for (variant = 0, variant_ptr = variant_list; - variant < total_variants; + for (variant = 0, variant_ptr = variant_list; variant < total_variants; variant++, variant_ptr++) { variant_ptr->avg_times = (uint64_t *) malloc(at_size); if (!variant_ptr->avg_times) { @@ -3193,9 +3111,8 @@ run_tests(void *arg) for (run = 0; run < NUM_RUNS; run++) { if (info->print_info) - fprintf(stderr, "\nStarting run %u of %d%c", - run + 1, NUM_RUNS, - silent_progress_bar ? '\r' : '\n' ); + fprintf(stderr, "\nStarting run %u of %d%c", run + 1, NUM_RUNS, + silent_progress_bar ? '\r' : '\n'); variant = 0; variant_ptr = variant_list; @@ -3212,12 +3129,12 @@ run_tests(void *arg) params.hash_alg = custom_job_params.hash_alg; /* Performing tests for each selected architecture */ - for (arch = ARCH_SSE; arch < NUM_ARCHS; arch++) { +#ifndef __aarch64__ + for (arch = ARCH_SSE; arch <= ARCH_AVX512; arch++) { if (archs[arch] == 0) continue; switch (arch) { -#ifdef __x86_64__ case ARCH_SSE: init_mb_mgr_sse(p_mgr); break; @@ -3227,23 +3144,28 @@ run_tests(void *arg) case ARCH_AVX2: init_mb_mgr_avx2(p_mgr); break; - case ARCH_AVX512: + default: /* ARCH_AV512 */ init_mb_mgr_avx512(p_mgr); break; -#endif /* __x86_64__ */ + } +#else + for (arch = ARCH_AARCH64; arch <= ARCH_SVE256; arch++) { + if (archs[arch] == 0) + continue; + + switch (arch) { -#ifdef __aarch64__ case ARCH_AARCH64: init_mb_mgr_aarch64(p_mgr); break; case ARCH_SVE256: init_mb_mgr_aarch64_sve256(p_mgr); break; -#endif /* __aarch64__ */ default: - fprintf(stderr, "Invalid architecture: %d\n", arch); - goto exit_failure; + init_mb_mgr_aarch64(p_mgr); + break; } +#endif /* __aarch64__ */ if (imb_get_errno(p_mgr) != 0) { printf("Error initializing MB_MGR! %s\n", @@ -3251,8 +3173,7 @@ run_tests(void *arg) goto exit_failure; } - process_variant(p_mgr, arch, ¶ms, - variant_ptr, run, buf, keys); + process_variant(p_mgr, arch, ¶ms, variant_ptr, run, buf, keys); /* update and print progress bar */ if (info->print_info) @@ -3297,9 +3218,17 @@ exit_failure: exit(EXIT_FAILURE); } -static void usage(void) +/* + * ========================================================= + * Utility functions + * ========================================================= + */ + +static void +usage(void) { - fprintf(stderr, "Usage: ipsec_perf [ARGS]\n" + fprintf(stderr, + "Usage: imb-perf [ARGS]\n" "\nALGORITHM can be one or more of:\n" "--cipher-algo: Select cipher algorithm to run on the custom test\n" "--hash-algo: Select hash algorithm to run on the custom test\n" @@ -3310,19 +3239,27 @@ static void usage(void) "-h: print this message\n" "-c: Use cold cache, it uses warm as default\n" "-w: Use warm cache\n" - "--arch: run only tests on specified architecture (SSE/AVX/AVX2/AVX512/AARCH64/SVE)\n" +#ifndef __aarch64__ + "--arch: run only tests on specified architecture (SSE/AVX/AVX2/AVX512)\n" +#else + "--arch: run only tests on specified architecture (AARCH64/SVE)\n" +#endif "--arch-best: detect available architectures and run only on the best one\n" "--cipher-dir: Select cipher direction to run on the custom test " "(encrypt/decrypt) (default = encrypt)\n" +#ifndef __aarch64__ "-o val: Use for the SHA size increment, default is 24\n" "--shani-on: use SHA extensions, default: auto-detect\n" "--shani-off: don't use SHA extensions\n" "--gfni-on: use Galois Field extensions, default: auto-detect\n" "--gfni-off: don't use Galois Field extensions\n" +#endif "--force-job-api: use JOB API" +#ifndef __aarch64__ " (direct API used for GCM/GHASH/CHACHA20_POLY1305 API by default)\n" "--gcm-sgl-api: use direct SGL API for GCM perf tests" " (direct GCM API is default)\n" +#endif "--threads num: for the number of threads to run" " Max: %d\n" "--cores mask: CPU's to run threads\n" @@ -3346,7 +3283,9 @@ static void usage(void) " a series of job sizes where on average 4 out of 10\n" " packets will be 64B long and 6 out of 10 packets\n" " will be 128B long)\n" +#ifndef __aarch64__ "--aad-size: size of AAD for AEAD algorithms\n" +#endif "--job-iter: number of tests iterations for each job size\n" "--no-progress-bar: Don't display progress bar\n" "--print-info: Display system and algorithm information\n" @@ -3360,13 +3299,16 @@ static void usage(void) "--burst-api: use burst API for perf tests\n" "--cipher-burst-api: use cipher-only burst API for perf tests\n" "--hash-burst-api: use hash-only burst API for perf tests\n" - "--burst-size: number of jobs to submit per burst\n", + "--burst-size: number of jobs to submit per burst\n" + "--quic-api: run QUIC-API specific tests only\n" + "--buffer-offset val: val is 0 by default, valid range is 0 to 15.\n" + " This option allows to test unaligned buffer cases\n", MAX_NUM_THREADS + 1); } static int -get_next_num_arg(const char * const *argv, const int index, const int argc, - void *dst, const size_t dst_size) +get_next_num_arg(const char *const *argv, const int index, const int argc, void *dst, + const size_t dst_size) { char *endptr = NULL; uint64_t val; @@ -3387,27 +3329,26 @@ get_next_num_arg(const char * const *argv, const int index, const int argc, val = strtoull(argv[index + 1], &endptr, 0); #endif if (endptr == argv[index + 1] || (endptr != NULL && *endptr != '\0')) { - fprintf(stderr, "Error converting '%s' as value for '%s'!\n", - argv[index + 1], argv[index]); + fprintf(stderr, "Error converting '%s' as value for '%s'!\n", argv[index + 1], + argv[index]); exit(EXIT_FAILURE); } switch (dst_size) { case (sizeof(uint8_t)): - *((uint8_t *)dst) = (uint8_t) val; + *((uint8_t *) dst) = (uint8_t) val; break; case (sizeof(uint16_t)): - *((uint16_t *)dst) = (uint16_t) val; + *((uint16_t *) dst) = (uint16_t) val; break; case (sizeof(uint32_t)): - *((uint32_t *)dst) = (uint32_t) val; + *((uint32_t *) dst) = (uint32_t) val; break; case (sizeof(uint64_t)): - *((uint64_t *)dst) = val; + *((uint64_t *) dst) = val; break; default: - fprintf(stderr, "%s() invalid dst_size %u!\n", - __func__, (unsigned) dst_size); + fprintf(stderr, "%s() invalid dst_size %u!\n", __func__, (unsigned) dst_size); exit(EXIT_FAILURE); break; } @@ -3418,14 +3359,15 @@ get_next_num_arg(const char * const *argv, const int index, const int argc, static int detect_arch(unsigned int arch_support[NUM_ARCHS]) { - const uint64_t detect_sse = - IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; - const uint64_t detect_avx = - IMB_FEATURE_AVX | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; +#ifndef __aarch64__ + const uint64_t detect_sse = IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; + const uint64_t detect_avx = IMB_FEATURE_AVX | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; const uint64_t detect_avx2 = IMB_FEATURE_AVX2 | detect_avx; const uint64_t detect_avx512 = IMB_FEATURE_AVX512_SKX | detect_avx2; +#else const uint64_t detect_aarch64 = IMB_FEATURE_AARCH64 | IMB_FEATURE_AESNI; const uint64_t detect_sve256 = IMB_FEATURE_AARCH64 | IMB_FEATURE_SVE256; +#endif IMB_MGR *p_mgr = NULL; enum arch_type_e arch_id; @@ -3434,7 +3376,11 @@ detect_arch(unsigned int arch_support[NUM_ARCHS]) return -1; } +#ifndef __aarch64__ for (arch_id = ARCH_SSE; arch_id < NUM_ARCHS; arch_id++) +#else + for (arch_id = ARCH_AARCH64; arch_id < NUM_ARCHS; arch_id++) +#endif arch_support[arch_id] = 1; p_mgr = alloc_mb_mgr(0); @@ -3443,6 +3389,7 @@ detect_arch(unsigned int arch_support[NUM_ARCHS]) return -1; } +#ifndef __aarch64__ if ((p_mgr->features & detect_avx512) != detect_avx512) arch_support[ARCH_AVX512] = 0; @@ -3454,12 +3401,13 @@ detect_arch(unsigned int arch_support[NUM_ARCHS]) if ((p_mgr->features & detect_sse) != detect_sse) arch_support[ARCH_SSE] = 0; - +#else if ((p_mgr->features & detect_aarch64) != detect_aarch64) arch_support[ARCH_AARCH64] = 0; if ((p_mgr->features & detect_sve256) != detect_sve256) arch_support[ARCH_SVE256] = 0; +#endif free_mb_mgr(p_mgr); @@ -3471,8 +3419,7 @@ detect_arch(unsigned int arch_support[NUM_ARCHS]) * with it. */ static const union params * -check_string_arg(const char *param, const char *arg, - const struct str_value_mapping *map, +check_string_arg(const char *param, const char *arg, const struct str_value_mapping *map, const unsigned int num_avail_opts) { unsigned int i; @@ -3498,8 +3445,8 @@ exit: } static int -parse_list(const char * const *argv, const int index, const int argc, - uint32_t *list, uint32_t *min, uint32_t *max) +parse_list(const char *const *argv, const int index, const int argc, uint32_t *list, uint32_t *min, + uint32_t *max) { char *token; uint32_t number; @@ -3542,8 +3489,7 @@ parse_list(const char * const *argv, const int index, const int argc, while (token != NULL) { if (count == MAX_LIST) { - fprintf(stderr, "Using only the first %d sizes\n", - MAX_LIST); + fprintf(stderr, "Using only the first %d sizes\n", MAX_LIST); break; } @@ -3577,14 +3523,13 @@ err_list: } static int -parse_range(const char * const *argv, const int index, const int argc, +parse_range(const char *const *argv, const int index, const int argc, uint32_t range_values[NUM_RANGE]) { char *token; uint32_t number; unsigned int i; - if (range_values == NULL || argv == NULL || index < 0 || argc < 0) { fprintf(stderr, "%s() internal error!\n", __func__); exit(EXIT_FAILURE); @@ -3624,7 +3569,7 @@ parse_range(const char * const *argv, const int index, const int argc, if (range_values[RANGE_MAX] < range_values[RANGE_MIN]) { fprintf(stderr, "Maximum value of range cannot be lower " - "than minimum value\n"); + "than minimum value\n"); exit(EXIT_FAILURE); } @@ -3636,13 +3581,11 @@ parse_range(const char * const *argv, const int index, const int argc, goto end_range; no_range: /* Try parsing as a list/single value */ - job_size_count = parse_list(argv, index, argc, job_size_list, - &job_sizes[RANGE_MIN], + job_size_count = parse_list(argv, index, argc, job_size_list, &job_sizes[RANGE_MIN], &job_sizes[RANGE_MAX]); end_range: free(copy_arg); return (index + 1); - } /** @@ -3656,14 +3599,15 @@ end_range: static int detect_best_arch(uint8_t arch_support[NUM_ARCHS]) { - const uint64_t detect_sse = - IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; - const uint64_t detect_avx = - IMB_FEATURE_AVX | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; +#ifndef __aarch64__ + const uint64_t detect_sse = IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; + const uint64_t detect_avx = IMB_FEATURE_AVX | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; const uint64_t detect_avx2 = IMB_FEATURE_AVX2 | detect_avx; const uint64_t detect_avx512 = IMB_FEATURE_AVX512_SKX | detect_avx2; +#else const uint64_t detect_aarch64 = IMB_FEATURE_AARCH64 | IMB_FEATURE_AESNI; const uint64_t detect_sve256 = IMB_FEATURE_AARCH64 | IMB_FEATURE_SVE256; +#endif IMB_MGR *p_mgr = NULL; uint64_t detected_features = 0; @@ -3684,6 +3628,7 @@ detect_best_arch(uint8_t arch_support[NUM_ARCHS]) memset(arch_support, 0, NUM_ARCHS * sizeof(arch_support[0])); +#ifndef __aarch64__ if ((detected_features & detect_avx512) == detect_avx512) { arch_support[ARCH_AVX512] = 1; return 0; @@ -3703,16 +3648,18 @@ detect_best_arch(uint8_t arch_support[NUM_ARCHS]) arch_support[ARCH_SSE] = 1; return 0; } +#else + if ((detected_features & detect_sve256) == detect_sve256) { + arch_support[ARCH_SVE256] = 1; + return 0; + } if ((detected_features & detect_aarch64) == detect_aarch64) { arch_support[ARCH_AARCH64] = 1; return 0; } +#endif - if ((detected_features & detect_sve256) == detect_sve256) { - arch_support[ARCH_SVE256] = 1; - return 0; - } fprintf(stderr, "Arch detection: no architecture available!\n"); return -1; } @@ -3720,7 +3667,8 @@ detect_best_arch(uint8_t arch_support[NUM_ARCHS]) /** * @brief Print system and application information */ -static void print_info(void) +static void +print_info(void) { uint32_t i; uint32_t supported_archs[NUM_ARCHS]; @@ -3742,8 +3690,7 @@ static void print_info(void) for (i = 0; i < DIM(arch_str_map); i++) if (arch_tab[i]) { - printf("Best architecture: %s\n", - arch_str_map[i].name); + printf("Best architecture: %s\n", arch_str_map[i].name); break; } @@ -3758,20 +3705,22 @@ static void print_info(void) printf("%s ", hash_algo_str_map[i].name); printf("\n"); -#ifdef __x86_64__ printf("Supported aead algorithms: "); +#ifndef __aarch64__ for (i = 0; i < DIM(aead_algo_str_map); i++) printf("%s ", aead_algo_str_map[i].name); - printf("\n"); #endif + printf("\n"); + return; - print_info_err: +print_info_err: fprintf(stderr, "%s() error!\n", __func__); exit(EXIT_FAILURE); } -int main(int argc, char *argv[]) +int +main(int argc, char *argv[]) { uint32_t num_t = 0; int i, core = 0; @@ -3823,8 +3772,7 @@ int main(int argc, char *argv[]) } else if (strcmp(argv[i], "--plot") == 0) { plot_output_option = 1; } else if (strcmp(argv[i], "--arch") == 0) { - values = check_string_arg(argv[i], argv[i+1], - arch_str_map, + values = check_string_arg(argv[i], argv[i + 1], arch_str_map, DIM(arch_str_map)); if (values == NULL) return EXIT_FAILURE; @@ -3840,107 +3788,87 @@ int main(int argc, char *argv[]) if (detect_best_arch(archs) != 0) return EXIT_FAILURE; } else if (strcmp(argv[i], "--cipher-algo") == 0) { - values = check_string_arg(argv[i], argv[i+1], - cipher_algo_str_map, - DIM(cipher_algo_str_map)); + values = check_string_arg(argv[i], argv[i + 1], cipher_algo_str_map, + DIM(cipher_algo_str_map)); if (values == NULL) return EXIT_FAILURE; - custom_job_params.cipher_mode = - values->job_params.cipher_mode; - custom_job_params.aes_key_size = - values->job_params.aes_key_size; + custom_job_params.cipher_mode = values->job_params.cipher_mode; + custom_job_params.aes_key_size = values->job_params.aes_key_size; cipher_algo_set = 1; i++; } else if (strcmp(argv[i], "--cipher-dir") == 0) { - values = check_string_arg(argv[i], argv[i+1], - cipher_dir_str_map, - DIM(cipher_dir_str_map)); + values = check_string_arg(argv[i], argv[i + 1], cipher_dir_str_map, + DIM(cipher_dir_str_map)); if (values == NULL) return EXIT_FAILURE; - custom_job_params.cipher_dir = - values->job_params.cipher_dir; + custom_job_params.cipher_dir = values->job_params.cipher_dir; cipher_dir_set = 1; i++; } else if (strcmp(argv[i], "--hash-algo") == 0) { - values = check_string_arg(argv[i], argv[i+1], - hash_algo_str_map, - DIM(hash_algo_str_map)); + values = check_string_arg(argv[i], argv[i + 1], hash_algo_str_map, + DIM(hash_algo_str_map)); if (values == NULL) return EXIT_FAILURE; - custom_job_params.hash_alg = - values->job_params.hash_alg; + custom_job_params.hash_alg = values->job_params.hash_alg; hash_algo_set = 1; i++; } else if (strcmp(argv[i], "--aead-algo") == 0) { - values = check_string_arg(argv[i], argv[i+1], - aead_algo_str_map, - DIM(aead_algo_str_map)); + values = check_string_arg(argv[i], argv[i + 1], aead_algo_str_map, + DIM(aead_algo_str_map)); if (values == NULL) return EXIT_FAILURE; - custom_job_params.cipher_mode = - values->job_params.cipher_mode; - custom_job_params.aes_key_size = - values->job_params.aes_key_size; - custom_job_params.hash_alg = - values->job_params.hash_alg; + custom_job_params.cipher_mode = values->job_params.cipher_mode; + custom_job_params.aes_key_size = values->job_params.aes_key_size; + custom_job_params.hash_alg = values->job_params.hash_alg; aead_algo_set = 1; i++; } else if (strcmp(argv[i], "-o") == 0) { - i = get_next_num_arg((const char * const *)argv, i, - argc, &sha_size_incr, + i = get_next_num_arg((const char *const *) argv, i, argc, &sha_size_incr, sizeof(sha_size_incr)); } else if (strcmp(argv[i], "--job-size") == 0) { /* Try parsing the argument as a range first */ - i = parse_range((const char * const *)argv, i, argc, - job_sizes); + i = parse_range((const char *const *) argv, i, argc, job_sizes); if (job_sizes[RANGE_MAX] > JOB_SIZE_TOP) { - fprintf(stderr, - "Invalid job size %u (max %d)\n", - (unsigned) job_sizes[RANGE_MAX], - JOB_SIZE_TOP); + fprintf(stderr, "Invalid job size %u (max %d)\n", + (unsigned) job_sizes[RANGE_MAX], JOB_SIZE_TOP); return EXIT_FAILURE; } } else if (strcmp(argv[i], "--imix") == 0) { - imix_list_count = parse_list((const char * const *)argv, - i, argc, imix_list, NULL, NULL); + imix_list_count = parse_list((const char *const *) argv, i, argc, imix_list, + NULL, NULL); if (imix_list_count == 0) { - fprintf(stderr, - "Invalid IMIX distribution list\n"); + fprintf(stderr, "Invalid IMIX distribution list\n"); return EXIT_FAILURE; } i++; } else if (strcmp(argv[i], "--aad-size") == 0) { /* Get AAD size for both GCM and CCM */ - i = get_next_num_arg((const char * const *)argv, i, - argc, &gcm_aad_size, + i = get_next_num_arg((const char *const *) argv, i, argc, &gcm_aad_size, sizeof(gcm_aad_size)); if (gcm_aad_size > AAD_SIZE_MAX) { - fprintf(stderr, - "Invalid AAD size %u (max %d)!\n", - (unsigned) gcm_aad_size, - AAD_SIZE_MAX); + fprintf(stderr, "Invalid AAD size %u (max %d)!\n", + (unsigned) gcm_aad_size, AAD_SIZE_MAX); return EXIT_FAILURE; } ccm_aad_size = gcm_aad_size; chacha_poly_aad_size = gcm_aad_size; snow_v_aad_size = gcm_aad_size; } else if (strcmp(argv[i], "--job-iter") == 0) { - i = get_next_num_arg((const char * const *)argv, i, - argc, &job_iter, sizeof(job_iter)); + i = get_next_num_arg((const char *const *) argv, i, argc, &job_iter, + sizeof(job_iter)); } else if (strcmp(argv[i], "--threads") == 0) { - i = get_next_num_arg((const char * const *)argv, i, - argc, &num_t, sizeof(num_t)); + i = get_next_num_arg((const char *const *) argv, i, argc, &num_t, + sizeof(num_t)); if (num_t > (MAX_NUM_THREADS + 1)) { fprintf(stderr, "Invalid number of threads!\n"); return EXIT_FAILURE; } } else if (strcmp(argv[i], "--cores") == 0) { - i = get_next_num_arg((const char * const *)argv, i, - argc, &core_mask, + i = get_next_num_arg((const char *const *) argv, i, argc, &core_mask, sizeof(core_mask)); } else if (strcmp(argv[i], "--unhalted-cycles") == 0) { use_unhalted_cycles = 1; @@ -3954,8 +3882,8 @@ int main(int argc, char *argv[]) } else if (strcmp(argv[i], "--no-tsc-detect") == 0) { tsc_detect = 0; } else if (strcmp(argv[i], "--tag-size") == 0) { - i = get_next_num_arg((const char * const *)argv, i, - argc, &tag_size, sizeof(tag_size)); + i = get_next_num_arg((const char *const *) argv, i, argc, &tag_size, + sizeof(tag_size)); } else if (strcmp(argv[i], "--burst-api") == 0) { test_api = TEST_API_BURST; } else if (strcmp(argv[i], "--cipher-burst-api") == 0) { @@ -3963,25 +3891,39 @@ int main(int argc, char *argv[]) } else if (strcmp(argv[i], "--hash-burst-api") == 0) { test_api = TEST_API_HASH_BURST; } else if (strcmp(argv[i], "--burst-size") == 0) { - i = get_next_num_arg((const char * const *)argv, i, - argc, &burst_size, + i = get_next_num_arg((const char *const *) argv, i, argc, &burst_size, sizeof(burst_size)); if (burst_size > (MAX_BURST_SIZE)) { - fprintf(stderr, "Burst size cannot be " - "more than %d\n", MAX_BURST_SIZE); + fprintf(stderr, + "Burst size cannot be " + "more than %d\n", + MAX_BURST_SIZE); return EXIT_FAILURE; } } else if (strcmp(argv[i], "--segment-size") == 0) { - i = get_next_num_arg((const char * const *)argv, i, - argc, &segment_size, + i = get_next_num_arg((const char *const *) argv, i, argc, &segment_size, sizeof(segment_size)); if (segment_size > (JOB_SIZE_TOP)) { - fprintf(stderr, "Segment size cannot be " - "more than %d\n", JOB_SIZE_TOP); + fprintf(stderr, + "Segment size cannot be " + "more than %d\n", + JOB_SIZE_TOP); return EXIT_FAILURE; } } else if (strcmp(argv[i], "--no-time-box") == 0) { use_timebox = 0; + } else if (strcmp(argv[i], "--quic-api") == 0) { + quic_api_test = 1; + } else if (strcmp(argv[i], "--buffer-offset") == 0) { + i = get_next_num_arg((const char *const *) argv, i, argc, &buffer_offset, + sizeof(buffer_offset)); + if (buffer_offset > 15) { + fprintf(stderr, + "Invalid --buffer-offset %u setting." + "Valid range is 0 to 15!\n", + (unsigned) buffer_offset); + return EXIT_FAILURE; + } } else { usage(); return EXIT_FAILURE; @@ -3989,8 +3931,8 @@ int main(int argc, char *argv[]) if (burst_size != 0 && test_api == TEST_API_JOB) { fprintf(stderr, "--burst-size can only be used with " - "--burst-api, --cipher-burst-api or " - "--hash-burst-api options\n"); + "--burst-api, --cipher-burst-api or " + "--hash-burst-api options\n"); return EXIT_FAILURE; } @@ -3998,11 +3940,10 @@ int main(int argc, char *argv[]) burst_size = DEFAULT_BURST_SIZE; /* currently only AES-CBC & CTR supported by cipher-only burst API */ - if (test_api == TEST_API_CIPHER_BURST && - (custom_job_params.cipher_mode != TEST_CBC && - custom_job_params.cipher_mode != TEST_CNTR)) { + if (test_api == TEST_API_CIPHER_BURST && (custom_job_params.cipher_mode != TEST_CBC && + custom_job_params.cipher_mode != TEST_CNTR)) { fprintf(stderr, "Unsupported cipher-only burst " - "API algorithm selected\n"); + "API algorithm selected\n"); return EXIT_FAILURE; } @@ -4013,23 +3954,22 @@ int main(int argc, char *argv[]) (custom_job_params.hash_alg != TEST_SHA_256_HMAC) && (custom_job_params.hash_alg != TEST_SHA_384_HMAC) && (custom_job_params.hash_alg != TEST_SHA_512_HMAC))) { - fprintf(stderr, - "Unsupported hash-only burst API algorithm selected\n"); + fprintf(stderr, "Unsupported hash-only burst API algorithm selected\n"); return EXIT_FAILURE; } - if (aead_algo_set == 0 && cipher_algo_set == 0 && - hash_algo_set == 0) { + if (aead_algo_set == 0 && cipher_algo_set == 0 && hash_algo_set == 0 && + quic_api_test == 0) { fprintf(stderr, "No cipher, hash or " - "AEAD algorithms selected\n"); + "AEAD algorithms selected\n"); usage(); return EXIT_FAILURE; } if (aead_algo_set && (cipher_algo_set || hash_algo_set)) { fprintf(stderr, "AEAD algorithm cannot be used " - "combined with another cipher/hash " - "algorithm\n"); + "combined with another cipher/hash " + "algorithm\n"); return EXIT_FAILURE; } @@ -4041,23 +3981,22 @@ int main(int argc, char *argv[]) if (custom_job_params.cipher_mode == TEST_CCM) { if (ccm_aad_size > CCM_AAD_SIZE_MAX) { - fprintf(stderr, "AAD cannot be higher than %d in CCM\n", - CCM_AAD_SIZE_MAX); + fprintf(stderr, "AAD cannot be higher than %d in CCM\n", CCM_AAD_SIZE_MAX); return EXIT_FAILURE; } } + srand(ITER_SCALE_LONG + ITER_SCALE_SHORT + ITER_SCALE_SMOKE); + if ((imix_list_count != 0)) { if (imix_list_count != job_size_count) { - fprintf(stderr, - "IMIX distribution list must have the same " - "number of items as the job list\n"); + fprintf(stderr, "IMIX distribution list must have the same " + "number of items as the job list\n"); return EXIT_FAILURE; } - job_size_imix_list = malloc(JOB_SIZE_IMIX_LIST*4); + job_size_imix_list = malloc(JOB_SIZE_IMIX_LIST * 4); if (job_size_imix_list == NULL) { - fprintf(stderr, - "Memory allocation for IMIX list failed\n"); + fprintf(stderr, "Memory allocation for IMIX list failed\n"); return EXIT_FAILURE; } @@ -4067,17 +4006,13 @@ int main(int argc, char *argv[]) * probabilities per job size */ distribution_total[0] = imix_list[0]; - for (i = 1; i < (int)imix_list_count; i++) - distribution_total[i] = imix_list[i] + - distribution_total[i-1]; + for (i = 1; i < (int) imix_list_count; i++) + distribution_total[i] = imix_list[i] + distribution_total[i - 1]; - /* Use always same seed */ - srand(0); /* Calculate a random sequence of packet sizes, based on distribution */ - for (i = 0; i < (int)JOB_SIZE_IMIX_LIST; i++) { - uint16_t random_number = rand() % - distribution_total[imix_list_count - 1]; + for (i = 0; i < (int) JOB_SIZE_IMIX_LIST; i++) { + uint16_t random_number = rand() % distribution_total[imix_list_count - 1]; uint16_t j; for (j = 0; j < imix_list_count; j++) @@ -4088,27 +4023,22 @@ int main(int argc, char *argv[]) } /* Calculate average buffer size for the IMIX distribution */ - for (i = 0; i < (int)imix_list_count; i++) - average_job_size += job_size_list[i] * - imix_list[i]; + for (i = 0; i < (int) imix_list_count; i++) + average_job_size += job_size_list[i] * imix_list[i]; - average_job_size /= - distribution_total[imix_list_count - 1]; + average_job_size /= distribution_total[imix_list_count - 1]; } - cipher_size_list = (uint32_t *) malloc(sizeof(uint32_t) * - num_sizes_list); + cipher_size_list = (uint32_t *) malloc(sizeof(uint32_t) * num_sizes_list); if (cipher_size_list == NULL) { fprintf(stderr, "Could not malloc cipher size list\n"); exit(EXIT_FAILURE); } - hash_size_list = (uint32_t *) malloc(sizeof(uint32_t) * - num_sizes_list); + hash_size_list = (uint32_t *) malloc(sizeof(uint32_t) * num_sizes_list); if (hash_size_list == NULL) { fprintf(stderr, "Could not malloc hash size list\n"); exit(EXIT_FAILURE); } - xgem_hdr_list = (uint64_t *) malloc(sizeof(uint64_t) * - num_sizes_list); + xgem_hdr_list = (uint64_t *) malloc(sizeof(uint64_t) * num_sizes_list); if (xgem_hdr_list == NULL) { fprintf(stderr, "Could not malloc xgem hdr list\n"); exit(EXIT_FAILURE); @@ -4116,13 +4046,14 @@ int main(int argc, char *argv[]) if (job_sizes[RANGE_MIN] == 0 && aead_algo_set == 0) { fprintf(stderr, "Buffer size cannot be 0 unless only " - "an AEAD algorithm is tested\n"); + "an AEAD algorithm is tested\n"); return EXIT_FAILURE; } /* Check num cores >= number of threads */ if ((core_mask != 0 && num_t != 0) && (num_t > bitcount(core_mask))) { - fprintf(stderr, "Insufficient number of cores in " + fprintf(stderr, + "Insufficient number of cores in " "core mask (0x%lx) to run %u threads!\n", (unsigned long) core_mask, num_t); return EXIT_FAILURE; @@ -4130,10 +4061,9 @@ int main(int argc, char *argv[]) /* Check timebox option vs number of threads bigger than 1 */ if (use_timebox && num_t > 1) { - fprintf(stderr, - "Time-box feature, enabled by default, doesn't work " - "safely with number of threads bigger than one! Please " - "use '--no-time-box' option to disable\n"); + fprintf(stderr, "Time-box feature, enabled by default, doesn't work " + "safely with number of threads bigger than one! Please " + "use '--no-time-box' option to disable\n"); return EXIT_FAILURE; } @@ -4141,7 +4071,7 @@ int main(int argc, char *argv[]) if (use_unhalted_cycles) { if (core_mask == 0) { fprintf(stderr, "Must specify core mask " - "when reading unhalted cycles!\n"); + "when reading unhalted cycles!\n"); return EXIT_FAILURE; } @@ -4158,33 +4088,38 @@ int main(int argc, char *argv[]) for (arch_id = 0; arch_id < NUM_ARCHS; arch_id++) { if (archs[arch_id] == 1 && arch_support[arch_id] == 0) { archs[arch_id] = 0; - fprintf(stderr, - "%s not supported. Disabling %s tests\n", - arch_str_map[arch_id].name, - arch_str_map[arch_id].name); + fprintf(stderr, "%s not supported. Disabling %s tests\n", + arch_str_map[arch_id].name, arch_str_map[arch_id].name); } } -#ifdef __aarch64__ - /* The scale maybe less than 0.01 on AARCH64, so precision of .3f - is not enough. Use .6f instead of .3f */ - if (tsc_detect) - fprintf(stderr, "TSC scaling to core cycles: %.6f\n", - get_tsc_to_core_scale(turbo_enabled)); -#else + if (tsc_detect) fprintf(stderr, "TSC scaling to core cycles: %.3f\n", get_tsc_to_core_scale(turbo_enabled)); -#endif -#ifdef __aarch64__ - fprintf(stderr, "CNT frequency: %ld\n", read_cntfreq()); -#endif + /** + * if SHA size increment not specified by user, set to default value + * - 24 for CBC-HMAC variants, otherwise 0 + **/ + if (sha_size_incr == UINT32_MAX) { + if ((custom_job_params.cipher_mode == TEST_CBC) && + ((custom_job_params.hash_alg == TEST_SHA1_HMAC) || + (custom_job_params.hash_alg == TEST_SHA_224_HMAC) || + (custom_job_params.hash_alg == TEST_SHA_256_HMAC) || + (custom_job_params.hash_alg == TEST_SHA_384_HMAC) || + (custom_job_params.hash_alg == TEST_SHA_512_HMAC) || + (custom_job_params.hash_alg == TEST_MD5))) + sha_size_incr = 24; + else + sha_size_incr = 0; + } fprintf(stderr, "Authentication size = cipher size + %u\n" + "Buffer offset = %u\n" "Tool version: %s\n" "Library version: %s\n", - sha_size_incr, IMB_VERSION_STR, imb_get_version_str()); + sha_size_incr, buffer_offset, IMB_VERSION_STR, imb_get_version_str()); if (!use_job_api) fprintf(stderr, "API type: direct\n"); @@ -4197,12 +4132,12 @@ int main(int argc, char *argv[]) } if (custom_job_params.cipher_mode == TEST_GCM) - fprintf(stderr, "GCM AAD = %"PRIu64"\n", gcm_aad_size); + fprintf(stderr, "GCM AAD = %" PRIu64 "\n", gcm_aad_size); if (custom_job_params.cipher_mode == TEST_CCM) - fprintf(stderr, "CCM AAD = %"PRIu64"\n", ccm_aad_size); + fprintf(stderr, "CCM AAD = %" PRIu64 "\n", ccm_aad_size); -#ifdef __x86_64__ +#ifndef __aarch64__ if (archs[ARCH_SSE]) { IMB_MGR *p_mgr = alloc_mb_mgr(flags); @@ -4212,25 +4147,14 @@ int main(int argc, char *argv[]) } init_mb_mgr_sse(p_mgr); fprintf(stderr, "%s SHA extensions (shani) for SSE arch\n", - (p_mgr->features & IMB_FEATURE_SHANI) ? - "Using" : "Not using"); + (p_mgr->features & IMB_FEATURE_SHANI) ? "Using" : "Not using"); free_mb_mgr(p_mgr); } -#else - IMB_MGR *p_mgr = alloc_mb_mgr(flags); - - if (p_mgr == NULL) { - fprintf(stderr, "Error allocating MB_MGR structure!\n"); - return EXIT_FAILURE; - } - free_mb_mgr(p_mgr); -#endif /* __x86_64__ */ +#endif /* __aarch64__ */ memset(t_info, 0, sizeof(t_info)); init_offsets(cache_type); - srand(ITER_SCALE_LONG + ITER_SCALE_SHORT + ITER_SCALE_SMOKE); - #ifdef LINUX if (use_timebox) { /* set up timebox callback function */ @@ -4254,21 +4178,19 @@ int main(int argc, char *argv[]) /* Allocate MB manager for each thread */ thread_info_p->p_mgr = alloc_mb_mgr(flags); if (thread_info_p->p_mgr == NULL) { - fprintf(stderr, "Failed to allocate MB_MGR " + fprintf(stderr, + "Failed to allocate MB_MGR " "structure for thread %u!\n", - (unsigned)(n + 1)); + (unsigned) (n + 1)); exit(EXIT_FAILURE); } #ifdef _WIN32 - threads[n] = (HANDLE) - _beginthread(&run_tests, 0, - (void *)thread_info_p); + threads[n] = (HANDLE) _beginthread(&run_tests, 0, (void *) thread_info_p); #else pthread_attr_t attr; pthread_attr_init(&attr); - pthread_create(&tids[n], &attr, run_tests, - (void *)thread_info_p); + pthread_create(&tids[n], &attr, run_tests, (void *) thread_info_p); #endif } } @@ -4277,7 +4199,7 @@ int main(int argc, char *argv[]) thread_info_p->p_mgr = alloc_mb_mgr(flags); if (thread_info_p->p_mgr == NULL) { fprintf(stderr, "Failed to allocate MB_MGR " - "structure for main thread!\n"); + "structure for main thread!\n"); exit(EXIT_FAILURE); } if (core_mask) { @@ -4285,7 +4207,7 @@ int main(int argc, char *argv[]) thread_info_p->core = core; } - run_tests((void *)thread_info_p); + run_tests((void *) thread_info_p); if (num_t > 1) { uint32_t n; @@ -4293,8 +4215,7 @@ int main(int argc, char *argv[]) WaitForMultipleObjects(num_t, threads, FALSE, INFINITE); #endif for (n = 0; n < (num_t - 1); n++) { - fprintf(stderr, "Waiting on thread %u to finish...\n", - (unsigned)(n + 2)); + fprintf(stderr, "Waiting on thread %u to finish...\n", (unsigned) (n + 2)); #ifdef _WIN32 CloseHandle(threads[n]); #else diff --git a/perf/ipsec_perf_tool.py b/perf/ipsec_perf_tool.py index 34d4f60d10552798fad298e71083f4e25a1910f7..09f31c2f98dc3c754efd5f6957d1c91b48c5d6ed 100755 --- a/perf/ipsec_perf_tool.py +++ b/perf/ipsec_perf_tool.py @@ -2,7 +2,7 @@ """ ********************************************************************** - Copyright(c) 2021-2022, Intel Corporation All rights reserved. + Copyright(c) 2021-2023, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -62,7 +62,7 @@ class Variant: hash_alg=None, aead_alg=None, sizes=None, offset=None, cold_cache=False, shani_off=False, force_job_api=False, unhalted_cycles=False, quick_test=False, smoke_test=False, - imix=None, aad_size=None, job_iter=None, no_time_box=False): + imix=None, aad_size=None, job_iter=None, no_time_box=False, buffer_offset=None): """Build perf app command line""" global PERF_APP @@ -88,6 +88,7 @@ class Variant: self.aad_size = aad_size self.job_iter = job_iter self.no_time_box = no_time_box + self.buffer_offset = buffer_offset if self.arch is not None: self.cmd += ' --arch {}'.format(self.arch) @@ -152,6 +153,8 @@ class Variant: if self.job_iter is not None: self.cmd += ' --job-iter {}'.format(self.job_iter) + if self.buffer_offset is not None: + self.cmd += ' --buffer-offset {}'.format(self.buffer_offset) def run(self): """Run perf app and store output""" @@ -258,9 +261,9 @@ def init_global_vars(): # detect OS and select app name if platform.system() == 'Windows': - PERF_APP = 'ipsec_perf.exe' + PERF_APP = 'imb-perf.exe' else: - PERF_APP = 'ipsec_perf' + PERF_APP = 'imb-perf' def get_info(): @@ -359,7 +362,6 @@ def parse_args(): global QUIET cores = None directions = ['encrypt', 'decrypt'] - offset = 24 alg_types = ['cipher-only', 'hash-only', 'aead-only', 'cipher-hash-all'] parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, @@ -367,15 +369,20 @@ def parse_args(): "performance application enabling extended functionality") # parse and validate args - parser.add_argument("-a", "--arch", choices=['SSE', 'AVX', 'AVX2', 'AVX512'], - default=None, action='append', - help="set architecture to test (default tests all supported archs)") + if platform.machine() != 'aarch64': + parser.add_argument("-a", "--arch", choices=['SSE', 'AVX', 'AVX2', 'AVX512'], + default=None, action='append', + help="set architecture to test (default tests all supported archs)") + else: + parser.add_argument("-a", "--arch", choices=['AARCH64', 'SVE256'], + default=None, action='append', + help="set architecture to test (default tests all supported archs)") parser.add_argument("-c", "--cores", default=cores, help="list/range of cores e.g. 2-8 or 3,4,5") parser.add_argument("-d", "--direction", default=None, choices=directions, help="Cipher direction") - parser.add_argument("-o", "--offset", default=offset, type=int, - help="offset for the SHA size increment, default is 24") + parser.add_argument("-o", "--offset", default=None, type=int, + help="offset for the SHA size increment") parser.add_argument("-t", "--alg-type", default=None, action='append', choices=alg_types, help="algorithm types to test") parser.add_argument("-s", "--job-size", default=None, @@ -421,6 +428,8 @@ def parse_args(): help="number of tests iterations for each job size") parser.add_argument("--no-time-box", default=False, action='store_true', help="disables time box feature for single packet size test duration (100ms)") + parser.add_argument("--buffer-offset", default=None, type=int, + help="buffer start address offset value 0-15, default 0") args = parser.parse_args() @@ -459,7 +468,7 @@ def parse_args(): alg_types, args.job_size, args.cold_cache, args.arch_best, \ args.shani_off, args.force_job_api, args.unhalted_cycles, \ args.quick, args.smoke, args.imix, \ - args.aad_size, args.job_iter, args.no_time_box + args.aad_size, args.job_iter, args.no_time_box, args.buffer_offset def run_test(core=None): @@ -529,7 +538,7 @@ def main(): # parse command line args archs, cores, directions, offset, alg_types, sizes, cold_cache, arch_best, \ shani_off, force_job_api, unhalted_cycles, quick_test, smoke_test, \ - imix, aad_size, job_iter, no_time_box = parse_args() + imix, aad_size, job_iter, no_time_box, buffer_offset = parse_args() # validate requested archs are supported if arch_best is True: @@ -580,7 +589,8 @@ def main(): cold_cache=cold_cache, shani_off=shani_off, force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, - aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) + aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box, + buffer_offset=buffer_offset)) TOTAL_VARIANTS += 1 if 'hash-only' in alg_types: @@ -591,7 +601,8 @@ def main(): cold_cache=cold_cache, shani_off=shani_off, force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, - aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) + aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box, + buffer_offset=buffer_offset)) TOTAL_VARIANTS += 1 if 'aead-only' in alg_types: @@ -602,7 +613,8 @@ def main(): cold_cache=cold_cache, shani_off=shani_off, force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, - aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) + aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box, + buffer_offset=buffer_offset)) TOTAL_VARIANTS += 1 if 'cipher-hash-all' in alg_types: @@ -616,7 +628,8 @@ def main(): shani_off=shani_off, force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, aad_size=aad_size, - job_iter=job_iter, no_time_box=no_time_box)) + job_iter=job_iter, no_time_box=no_time_box, + buffer_offset=buffer_offset)) TOTAL_VARIANTS += 1 # take starting timestamp diff --git a/perf/misc.asm b/perf/misc.asm index 242cdb773ad75de3b5bfb5115828ae1a79dc47ab..c4d4cb7f7388501a649a6cb8a65c161d6623fc97 100644 --- a/perf/misc.asm +++ b/perf/misc.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2021-2022, Intel Corporation All rights reserved. +; Copyright(c) 2021-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions diff --git a/perf/misc.h b/perf/misc.h index 2fcdbe9a63a65501935077b35cc958481be12efa..e78b2d05bfefbc182d805952d27cbbc180b77be6 100644 --- a/perf/misc.h +++ b/perf/misc.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2021-2022, Intel Corporation + Copyright (c) 2021-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -33,9 +33,9 @@ * * @return Number of TSC cycles measured while in fixed cost loop */ -uint64_t measure_tsc(const uint64_t cycles); +uint64_t +measure_tsc(const uint64_t cycles); #ifdef __aarch64__ -uint64_t rdtscp(void); -uint64_t read_cntfreq(void); +uint64_t __rdtscp(void *); #endif diff --git a/perf/misc_aarch64.S b/perf/misc_aarch64.S index 3a3d112ba87bfc2d13cdd8b289a231c8e7dfe45a..3fa1131c7a6cf8807ff321f05d465d2628bcfc01 100644 --- a/perf/misc_aarch64.S +++ b/perf/misc_aarch64.S @@ -42,19 +42,10 @@ fixed_loop: ret -.global rdtscp -.type rdtscp,%function +.global __rdtscp +.type __rdtscp,%function .align 5 -rdtscp: +__rdtscp: mrs x0,CNTVCT_EL0 ret - - -.global read_cntfreq -.type read_cntfreq,%function -.align 5 -read_cntfreq: - mrs x0,CNTFRQ_EL0 - - ret diff --git a/perf/msr.c b/perf/msr.c index be4dfb4f2c6adc6da64d6bb9fbd1a603e43831f1..0ee76d9fcc9fe5082664b4337f17a1069aff65cd 100644 --- a/perf/msr.c +++ b/perf/msr.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2018-2022 Intel Corporation All rights reserved. + Copyright(c) 2018-2023 Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -35,6 +35,11 @@ #include #include +#ifdef _MSC_VER +/* disable C5105 warning produced by standard headers C11 C standard */ +#pragma warning(disable : 5105) +#endif + #ifndef _WIN32 #include #include @@ -51,9 +56,9 @@ #include "msr.h" -static int *m_msr_fd = NULL; /**< MSR driver file descriptors table */ -static unsigned m_maxcores = 0; /**< max number of cores (size of the - table above too) */ +static int *m_msr_fd = NULL; /**< MSR driver file descriptors table */ +static unsigned m_maxcores = 0; /**< max number of cores (size of the + table above too) */ #ifdef _WIN32 #ifdef WIN_MSR union msr_data { @@ -126,7 +131,7 @@ machine_init(const unsigned max_core_id) * Each file descriptor is for a different core. * Core id is an index to the table. */ - m_msr_fd = (int *)malloc(m_maxcores * sizeof(m_msr_fd[0])); + m_msr_fd = (int *) malloc(m_maxcores * sizeof(m_msr_fd[0])); if (m_msr_fd == NULL) { m_maxcores = 0; return MACHINE_RETVAL_ERROR; @@ -190,8 +195,7 @@ msr_file_open(const unsigned lcore) char fname[32]; memset(fname, 0, sizeof(fname)); - snprintf(fname, sizeof(fname)-1, - "/dev/cpu/%u/msr", lcore); + snprintf(fname, sizeof(fname) - 1, "/dev/cpu/%u/msr", lcore); fd = open(fname, O_RDWR); if (fd < 0) fprintf(stderr, "Error opening file '%s'!\n", fname); @@ -204,9 +208,7 @@ msr_file_open(const unsigned lcore) #endif /* _WIN32 */ int -msr_read(const unsigned lcore, - const uint32_t reg, - uint64_t *value) +msr_read(const unsigned lcore, const uint32_t reg, uint64_t *const value) { int ret = MACHINE_RETVAL_OK; #ifdef _WIN32 @@ -229,8 +231,7 @@ msr_read(const unsigned lcore, #ifdef _WIN32 #ifdef WIN_MSR msr.ui64 = 0; - status = RdmsrTx((DWORD)reg, &(msr.ui32.low), - &(msr.ui32.high), (1ULL << lcore)); + status = RdmsrTx((DWORD) reg, &(msr.ui32.low), &(msr.ui32.high), (1ULL << lcore)); if (status) *value = msr.ui64; else @@ -244,22 +245,19 @@ msr_read(const unsigned lcore, if (fd < 0) return MACHINE_RETVAL_ERROR; - read_ret = pread(fd, value, sizeof(value[0]), (off_t)reg); + read_ret = pread(fd, value, sizeof(value[0]), (off_t) reg); if (read_ret != sizeof(value[0])) ret = MACHINE_RETVAL_ERROR; #endif /* _WIN32 */ if (ret != MACHINE_RETVAL_OK) - fprintf(stderr, "RDMSR failed for reg[0x%x] on lcore %u\n", - (unsigned)reg, lcore); + fprintf(stderr, "RDMSR failed for reg[0x%x] on lcore %u\n", (unsigned) reg, lcore); return ret; } int -msr_write(const unsigned lcore, - const uint32_t reg, - const uint64_t value) +msr_write(const unsigned lcore, const uint32_t reg, const uint64_t value) { int ret = MACHINE_RETVAL_OK; #ifdef _WIN32 @@ -279,8 +277,7 @@ msr_write(const unsigned lcore, #ifdef _WIN32 #ifdef WIN_MSR msr.ui64 = value; - status = WrmsrTx((DWORD)reg, msr.ui32.low, - msr.ui32.high, (1ULL << lcore)); + status = WrmsrTx((DWORD) reg, msr.ui32.low, msr.ui32.high, (1ULL << lcore)); if (!status) ret = MACHINE_RETVAL_ERROR; #endif /* WIN_MSR */ @@ -292,15 +289,16 @@ msr_write(const unsigned lcore, if (fd < 0) return MACHINE_RETVAL_ERROR; - write_ret = pwrite(fd, &value, sizeof(value), (off_t)reg); + write_ret = pwrite(fd, &value, sizeof(value), (off_t) reg); if (write_ret != sizeof(value)) ret = MACHINE_RETVAL_ERROR; #endif /* _WIN32 */ if (ret != MACHINE_RETVAL_OK) - fprintf(stderr, "WRMSR failed for reg[0x%x] " + fprintf(stderr, + "WRMSR failed for reg[0x%x] " "<- value[0x%llx] on lcore %u\n", - (unsigned)reg, (unsigned long long)value, lcore); + (unsigned) reg, (unsigned long long) value, lcore); return ret; } diff --git a/perf/msr.h b/perf/msr.h index 6cc7c76ff332fb1e87d99cf8bd64881788324860..89489e7bda9b887ef34b77e6ac7335d6aee81afd 100644 --- a/perf/msr.h +++ b/perf/msr.h @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2018-2022 Intel Corporation All rights reserved. + Copyright(c) 2018-2023 Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -44,18 +44,17 @@ extern "C" { #endif - #ifdef DEBUG #define ASSERT assert #else #define ASSERT(x) #endif -#define MACHINE_DEFAULT_MAX_COREID 255 /**< max core id */ +#define MACHINE_DEFAULT_MAX_COREID 255 /**< max core id */ -#define MACHINE_RETVAL_OK 0 /**< everything OK */ -#define MACHINE_RETVAL_ERROR 1 /**< generic error */ -#define MACHINE_RETVAL_PARAM 2 /**< parameter error */ +#define MACHINE_RETVAL_OK 0 /**< everything OK */ +#define MACHINE_RETVAL_ERROR 1 /**< generic error */ +#define MACHINE_RETVAL_PARAM 2 /**< parameter error */ /** * @brief Initializes machine module @@ -67,7 +66,8 @@ extern "C" { * @return Operation status * @retval MACHINE_RETVAL_OK on success */ -int machine_init(const unsigned max_core_id); +int +machine_init(const unsigned max_core_id); /** * @brief Shuts down machine module @@ -75,7 +75,8 @@ int machine_init(const unsigned max_core_id); * @return Operation status * @retval MACHINE_RETVAL_OK on success */ -int machine_fini(void); +int +machine_fini(void); /** * @brief Executes RDMSR on \a lcore logical core @@ -88,9 +89,7 @@ int machine_fini(void); * @retval MACHINE_RETVAL_OK on success */ int -msr_read(const unsigned lcore, - const uint32_t reg, - uint64_t *value); +msr_read(const unsigned lcore, const uint32_t reg, uint64_t *const value); /** * @brief Executes WRMSR on \a lcore logical core @@ -103,9 +102,7 @@ msr_read(const unsigned lcore, * @retval MACHINE_RETVAL_OK on success */ int -msr_write(const unsigned lcore, - const uint32_t reg, - const uint64_t value); +msr_write(const unsigned lcore, const uint32_t reg, const uint64_t value); #ifdef __cplusplus } diff --git a/perf/win_x64.mak b/perf/win_x64.mak index 06165a383b10a48d9738e905fe966c7863f73019..9ab5dfa52c2bf1eaebe9a29ca00c2a2a5f71e949 100644 --- a/perf/win_x64.mak +++ b/perf/win_x64.mak @@ -1,5 +1,5 @@ # -# Copyright (c) 2017-2022, Intel Corporation +# Copyright (c) 2017-2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -APP = ipsec_perf +APP = imb-perf INSTNAME = ipsec-mb !if !defined(PREFIX) @@ -63,7 +63,7 @@ DLFLAGS = CC = cl # _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure strtok() being used -CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) +CFLAGS = /nologo /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) /std:c11 LNK = link LFLAGS = /out:$(APP).exe $(DLFLAGS) diff --git a/test/.clang-format b/test/.clang-format deleted file mode 100755 index 87bfece704907c41ee8a1a51f1363450e2643b6f..0000000000000000000000000000000000000000 --- a/test/.clang-format +++ /dev/null @@ -1,20 +0,0 @@ -BasedOnStyle: LLVM -IndentWidth: 8 -Language: Cpp -BreakBeforeBraces: Linux -AllowShortIfStatementsOnASingleLine: false -IndentCaseLabels: false -UseTab: Never -AlignConsecutiveMacros: true -AlignTrailingComments: true -AlwaysBreakAfterReturnType: AllDefinitions -SortIncludes: false -BreakBeforeInheritanceComma: true -AllowAllParametersOfDeclarationOnNextLine: false -BinPackParameters: false -BinPackArguments: true -ReflowComments: true -SpaceAfterCStyleCast: false -ColumnLimit: 80 -Cpp11BracedListStyle: false -MaxEmptyLinesToKeep: 1 \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a746466ef48d638a87dbfe5c5c9571398104371e --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,41 @@ +# Copyright (c) 2022-2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +# build imb-kat application +add_subdirectory(kat-app) + +# build imb-xvalid application +add_subdirectory(xvalid-app) + +# build imb-wycheproof application +if(NOT "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + add_subdirectory(wycheproof-app) +endif() + diff --git a/test/Makefile b/test/Makefile index 279e617593dca20d17b7930cc34e361af1236f94..b92352de1972af7c2142cc02d89057100aec5b31 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,5 +1,5 @@ # -# Copyright (c) 2012-2022, Intel Corporation +# Copyright (c) 2012-2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -25,197 +25,45 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TEST_APP := ipsec_MB_testapp -XVALID_APP := ipsec_xvalid_test -FUZZ_APP := job_api_fuzz_test -ACVP_APP := acvp_app -INSTPATH ?= /usr/include/ipsec-mb.h -LIB_DIR ?= ../lib +KAT_APP := kat-app +XVALID_APP := xvalid-app +FUZZ_APP := fuzz-app +ACVP_APP := acvp-app +WYCHEPROOF_APP := wycheproof-app -ARCH = $(shell uname -m) - -USE_YASM ?= n -YASM ?= yasm -NASM ?= nasm - -MINGW ?= $(shell $(CC) -dM -E - < /dev/null | grep -i mingw | wc -l | sed 's/^ *//') - -CFLAGS = -MMD -D_GNU_SOURCE -DNO_COMPAT_IMB_API_053 \ - -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ - -Wcast-qual -Wundef -Wwrite-strings \ - -Wformat -Wformat-security \ - -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ - -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ - -fno-delete-null-pointer-checks -fwrapv - -# -fno-strict-overflow is not supported by clang -ifneq ($(CC),clang) -CFLAGS += -fno-strict-overflow -endif - -# if "-z ibt" is supported then assume "-z shstk, -z cet-report=error" are also supported -# "-fcf-protection" needs to be checked separately -ifeq ($(ARCH),x86_64) -ifeq ($(MINGW),0) -CC_HAS_CET = $(and $(shell $(CC) --target-help 2> /dev/null | grep -m1 -e "-z ibt" | wc -l), \ - $(shell $(CC) --help=common 2> /dev/null | grep -m1 -e "-fcf-protection" | wc -l)) -CET_LDFLAGS=-r -z ibt -z shstk -endif # MINGW -endif # x86_64 - -ifeq ($(ARCH),x86_64) -ifeq ($(CC_HAS_CET),1) -CFLAGS += -fcf-protection=full -endif -endif +# targets come here +all: $(KAT_APP) $(XVALID_APP) $(FUZZ_APP) $(ACVP_APP) $(WYCHEPROOF_APP) -YASM_FLAGS := -f x64 -f elf64 -X gnu -g dwarf2 -DLINUX -D__linux__ -ifeq ($(MINGW),0) -CFLAGS += -DLINUX -NASM_FLAGS := -Werror -felf64 -Xgnu -gdwarf -DLINUX -D__linux__ -else -NASM_FLAGS := -Werror -fwin64 -Xvc -gcv8 -DWIN_ABI -endif +.PHONY: $(KAT_APP) $(XVALID_APP) $(FUZZ_APP) $(ACVP_APP) $(WYCHEPROOF_APP) -ifeq ($(MINGW),0) -LDFLAGS = -fPIE -z noexecstack -z relro -z now -else -LDFLAGS = -fPIE -endif +$(KAT_APP): + $(MAKE) -C kat-app -ifeq ($(ARCH),x86_64) -ifeq ($(CC_HAS_CET),1) -LDFLAGS += -fcf-protection=full -Wl,-z,ibt -Wl,-z,shstk -Wl,-z,cet-report=error -endif -endif -LDLIBS = -lIPSec_MB - -ifeq ("$(shell test -r $(INSTPATH) && echo -n yes)","yes") -# library installed -CFLAGS += -else -# library not installed -CFLAGS += -I../lib/include -I../lib -LDFLAGS += -L$(LIB_DIR) -endif +$(XVALID_APP): + $(MAKE) -C xvalid-app -FUZZ_CFLAGS = -I../lib -L../lib $(CFLAGS) -g -fsanitize=fuzzer,address,leak - -DEBUG_OPT ?= -O0 -ifeq ($(DEBUG),y) -CFLAGS += $(DEBUG_OPT) -DDEBUG -g -LDFLAGS += -g -else -ifeq ($(MINGW),0) -CFLAGS += -O3 -else -CFLAGS += -O2 -endif # MINGW -endif # DEBUG - -ACVP_LOC ?= /usr/local/acvp -ACVP_HDR ?= $(ACVP_LOC)/include -ACVP_LIB ?= $(ACVP_LOC)/lib -ACVP_CFLAGS = -I$(ACVP_HDR) $(CFLAGS) -ACVP_LDFLAGS = -L$(ACVP_LIB) $(LDFLAGS) -ACVP_LDLIBS = -lacvp $(LDLIBS) - -# ipsec_MB_testapp modules -ifeq ($(ARCH),aarch64) -SOURCES := main.c utils.c api_test.c snow3g_test.c direct_api_test.c clear_mem_test.c zuc_test.c -OBJECTS := $(SOURCES:%.c=%.o) - -# ipsec_xvalid_test modules -XVALID_ASM := misc_aarch64.S -XVALID_SOURCES := ipsec_xvalid.c utils.c -XVALID_OBJECTS := $(XVALID_SOURCES:%.c=%.o) $(XVALID_ASM:%.S=%.o) - -# fuzz modules -FUZZ_SOURCES := job_api_fuzz_test.c - -# list of present dependency files -DEP_FILES = $(wildcard ./*.d) - -# rule for compiling assembly code with producing dependencies -%.o:%.S - $(CC) -c $(CFLAGS) $< -o $@ -endif # aarch64 +$(FUZZ_APP): + $(MAKE) -C fuzz-app ifeq ($(ARCH),x86_64) -SOURCES := main.c gcm_test.c ctr_test.c customop_test.c des_test.c ccm_test.c \ - cmac_test.c utils.c hmac_sha1_test.c hmac_sha256_sha512_test.c \ - hmac_md5_test.c aes_test.c sha_test.c chained_test.c api_test.c pon_test.c \ - ecb_test.c zuc_test.c kasumi_test.c snow3g_test.c direct_api_test.c clear_mem_test.c \ - hec_test.c xcbc_test.c aes_cbcs_test.c crc_test.c chacha_test.c poly1305_test.c \ - chacha20_poly1305_test.c null_test.c snow_v_test.c direct_api_param_test.c -OBJECTS := $(SOURCES:%.c=%.o) - -ifneq ($(PIN_CEC_ROOT),) -CFLAGS += -I$(PIN_CEC_ROOT)/include -DPIN_BASED_CEC -endif - -# ipsec_xvalid_test modules -XVALID_ASM := misc.asm -XVALID_SOURCES := ipsec_xvalid.c utils.c -XVALID_OBJECTS := $(XVALID_SOURCES:%.c=%.o) $(XVALID_ASM:%.asm=%.o) - -# fuzz modules -FUZZ_SOURCES := job_api_fuzz_test.c - -# acvp_app modules -ACVP_SOURCES := acvp_app_main.c utils.c - -# list of present dependency files -DEP_FILES = $(wildcard ./*.d) - -# rule for compiling assembly code with producing dependencies -%.o:%.asm -ifeq ($(USE_YASM),y) - $(YASM) $(YASM_FLAGS) $< -o $@ -else - $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< -endif -ifeq ($(CC_HAS_CET),1) - $(LD) $(CET_LDFLAGS) -o $@.tmp $@ - mv $@.tmp $@ -endif # CC_HAS_CET -endif # x86_64 - -# targets come here -all: $(TEST_APP) $(XVALID_APP) $(FUZZ_APP) $(ACVP_APP) +$(ACVP_APP): + $(MAKE) -C acvp-app -$(TEST_APP): $(OBJECTS) - $(CC) $(LDFLAGS) $^ $(LDLIBS) -o $@ - -$(XVALID_APP): $(XVALID_OBJECTS) - $(CC) $(LDFLAGS) $^ $(LDLIBS) -o $@ - -$(ACVP_APP): $(ACVP_SOURCES) -ifneq ("$(wildcard $(ACVP_HDR))","") - $(CC) $(ACVP_CFLAGS) $(ACVP_LDFLAGS) $^ $(ACVP_LDLIBS) -o $@ -else - @echo "No libacvp header found at $(ACVP_HDR), $(ACVP_APP) is not built" -endif - -$(FUZZ_APP): $(FUZZ_SOURCES) -ifeq ($(MINGW),0) -ifneq (, $(shell which clang)) - clang $(FUZZ_CFLAGS) $(FUZZ_SOURCES) $(LDLIBS) -o $@ -else - @echo "Clang is not installed. $(FUZZ_APP) is not built" -endif -else - @echo "Fuzzing not supported on MinGW build. $(FUZZ_APP) is not built" +$(WYCHEPROOF_APP): + $(MAKE) -C wycheproof-app endif - .PHONY: clean clean: - -rm -f $(OBJECTS) $(TEST_APP) $(DEP_FILES) $(XVALID_OBJECTS) $(XVALID_APP) $(ASM_OBJECTS) $(FUZZ_APP) $(ACVP_APP) + $(MAKE) -C kat-app clean + $(MAKE) -C xvalid-app clean + $(MAKE) -C fuzz-app clean + $(MAKE) -C acvp-app clean + $(MAKE) -C wycheproof-app clean # style check section CHECKPATCH?=checkpatch.pl -CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,AVOID_EXTERNS,COMPLEX_MACRO,USE_FUNC,CONSTANT_COMPARISON,MISSING_SPACE,NEW_TYPEDEFS +CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --max-line-length=100 --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,AVOID_EXTERNS,COMPLEX_MACRO,USE_FUNC,CONSTANT_COMPARISON,MISSING_SPACE,NEW_TYPEDEFS %.c_style_check : %.c $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< @@ -229,15 +77,35 @@ CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --ignore CODE_INDEN %.inc_style_check : %.inc $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< -SOURCES_ALL := $(wildcard *.[ch]) $(wildcard *.asm) $(wildcard *.inc) +KAT_APP_SOURCES := $(wildcard $(KAT_APP)/*.[ch]) +FUZZ_APP_SOURCES := $(wildcard $(FUZZ_APP)/*.[ch]) +ACVP_APP_SOURCES := $(wildcard $(ACVP_APP)/*.[ch]) +WYCHEPROOF_APP_SOURCES := $(wildcard $(WYCHEPROOF_APP)/*.[ch]) +XVALID_APP_SOURCES := $(wildcard $(XVALID_APP)/*.[ch]) \ + $(wildcard $(XVALID_APP)/*.asm) \ + $(wildcard $(XVALID_APP)/*.inc) +SOURCES_ALL := $(KAT_APP_SOURCES) $(XVALID_APP_SOURCES) \ + $(FUZZ_APP_SOURCES) $(ACVP_APP_SOURCES) $(WYCHEPROOF_APP_SOURCES) SOURCES_STYLE := $(foreach infile,$(SOURCES_ALL),$(infile)_style_check) .PHONY: style style: $(SOURCES_STYLE) -# if target not clean or rinse then make dependencies -ifneq ($(MAKECMDGOALS),clean) -ifneq ($(MAKECMDGOALS),style) --include $(wildcard *.d) -endif -endif +# cppcheck analysis check +CPPCHECK ?= cppcheck +CPPCHECK_OPTS ?= -I./ -I../lib +CPPCHECK_FLAGS ?= -j $(shell getconf _NPROCESSORS_ONLN) +CPPCHECK_FLAGS1 ?= --cppcheck-build-dir=.cppcheck $(CPPCHECK_FLAGS) +CPPCHECK_FLAGS2 ?= --cppcheck-build-dir=.bughunt $(CPPCHECK_FLAGS) +.PHONY: cppcheck +cppcheck: + mkdir -p .cppcheck + $(CPPCHECK) --force --enable=all $(CPPCHECK_FLAGS1) $(CPPCHECK_OPTS) \ + ./kat-app ./xvalid-app ./fuzz-app ./acvp-app ./wycheproof-app + +.PHONY: bughunt +bughunt: + mkdir -p .bughunt + $(CPPCHECK) --bug-hunting --inconclusive $(CPPCHECK_FLAGS2) $(CPPCHECK_OPTS) \ + ./kat-app ./xvalid-app ./fuzz-app ./acvp-app ./wycheproof-app + diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4c8475fce044a29c38925c11e624f77edcf0172d --- /dev/null +++ b/test/README.md @@ -0,0 +1,24 @@ +# Intel(R) Multi-Buffer Crypto for IPsec Library - Test Applications + +## Contents + +- Overview +- Library installation + +## Overview + +The test directory contains multiple applications for testing the Intel(R) Multi-Buffer Crypto for IPsec Library. +Applications perform various types of testing such as Known Answer Tests (KAT), API input validation, fuzz testing and more. +Test application usage is documented in their README files, and assume the library is installed. +For instructions to install the library, see the Library Installation section below. + +## Library Installation + +### Linux +- Use standard OS installer to install the "intel-ipsec-mb" package (`apt`, `dnf` etc.) +- Alternatively, install the library from source by following the instructions in the [README](https://github.com/intel/intel-ipsec-mb/#8-installation) + + +### Windows +- Install the library from source following the instructions in the [README](https://github.com/intel/intel-ipsec-mb\#8-installation) + diff --git a/test/acvp-app/Makefile b/test/acvp-app/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..90d1c102cccc1d0f2de70ba9b58964ab81658d60 --- /dev/null +++ b/test/acvp-app/Makefile @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP := imb-acvp + +# include common options +include ../common/common.mk + +ACVP_LOC ?= /usr/local/acvp +ACVP_HDR ?= $(ACVP_LOC)/include +ACVP_LIB ?= $(ACVP_LOC)/lib +ACVP_CFLAGS = -I$(ACVP_HDR) $(CFLAGS) +ACVP_LDFLAGS = -L$(ACVP_LIB) $(LDFLAGS) +ACVP_LDLIBS = -lacvp $(LDLIBS) + +# acvp_app modules +ACVP_SOURCES := acvp_app_main.c utils.o + +# targets come here +all: $(APP) + +$(APP): $(ACVP_SOURCES) +ifneq ("$(wildcard $(ACVP_HDR))","") + $(CC) $(ACVP_CFLAGS) $(ACVP_LDFLAGS) $^ $(ACVP_LDLIBS) -o $@ +else + @echo "No libacvp header found at $(ACVP_HDR), $(APP) is not built" +endif + +utils.o: ../common/utils.c + $(CC) -c $(CFLAGS) $< -o $@ + +.PHONY: clean +clean: + -rm -f $(DEP_FILES) $(APP) + +# if target not clean then make dependencies +ifneq ($(MAKECMDGOALS),clean) +-include $(wildcard *.d) +endif diff --git a/test/acvp-app/README.md b/test/acvp-app/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7432c5a452baea165f6259ec28483e582a4420dc --- /dev/null +++ b/test/acvp-app/README.md @@ -0,0 +1,35 @@ +# Intel(R) Multi-Buffer Crypto for IPsec Library - ACVP Test Application + +## Contents + +- Overview +- Usage + + +## Overview + +The [ACVP](https://pages.nist.gov/ACVP/draft-fussell-acvp-spec.html) +(Automated Cryptographic Validation Protocol) test application performs +validation of NIST-approved cryptographic algorithms as part of the +[CAVP](https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program) +(Cryptographic Algorithm Validation Program). +[libacvp](https://github.com/cisco/libacvp) is used as the client-side implementation [ACVP protocol](github.com/usnistgov/ACVP). +See intel-ipsec-mb [README](https://github.com/intel/intel-ipsec-mb/blob/main/README.md#12-fips-compliance) for details about exercised algorithms and their parameters. + +## Usage + +Before running the application, ensure the library is installed by following the instructions +in the [README](https://github.com/intel/intel-ipsec-mb/tree/main/test#library-installation). +Installation of [libacvp](https://github.com/cisco/libacvp) is also required. Please follow its build and install instructions. +### Linux +To perform AES-GCM validation: +`./imb-acvp --req AES-GCM-req.json --resp AES-GCM-resp.json` + +To perform AES-GCM validation for AVX512 architecture only: +`./imb-acvp --req AES-GCM-req.json --resp AES-GCM-resp.json --arch AVX512` + +To display an extensive help page: +`./imb-acvp --help` + +### Windows +Not currently supported. diff --git a/test/acvp_app_main.c b/test/acvp-app/acvp_app_main.c similarity index 79% rename from test/acvp_app_main.c rename to test/acvp-app/acvp_app_main.c index 8f8ed0d0b140f09cbebab83c7574821667184e70..e98ee13b489cf7e44e922824c52a9fc232f567a0 100644 --- a/test/acvp_app_main.c +++ b/test/acvp-app/acvp_app_main.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2022, Intel Corporation All rights reserved. + Copyright(c) 2022-2023, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -55,14 +55,14 @@ static int aes_cbc_handler(ACVP_TEST_CASE *test_case) static uint8_t next_iv[16]; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.symmetric; if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { fprintf(stderr, "Unsupported direction\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } switch (tc->key_len) { @@ -77,7 +77,7 @@ static int aes_cbc_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } job = IMB_GET_NEXT_JOB(mb_mgr); @@ -112,7 +112,7 @@ static int aes_cbc_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } else /* DECRYPT */ { job->cipher_direction = IMB_DIR_DECRYPT; @@ -127,7 +127,7 @@ static int aes_cbc_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } /* @@ -137,7 +137,89 @@ static int aes_cbc_handler(ACVP_TEST_CASE *test_case) if (tc->test_type == ACVP_SYM_TEST_TYPE_MCT) memcpy(next_iv, tc->ct, 16); - return EXIT_SUCCESS; + return ACVP_SUCCESS; +} + +static int aes_ecb_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_SYM_CIPHER_TC *tc; + IMB_JOB *job = NULL; + DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); + DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); + + if (test_case == NULL) + return ACVP_CRYPTO_MODULE_FAIL; + + tc = test_case->tc.symmetric; + + if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && + tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { + fprintf(stderr, "Unsupported direction\n"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + switch (tc->key_len) { + case 128: + IMB_AES_KEYEXP_128(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 192: + IMB_AES_KEYEXP_192(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 256: + IMB_AES_KEYEXP_256(mb_mgr, tc->key, enc_keys, dec_keys); + break; + default: + fprintf(stderr, "Unsupported AES key length\n"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len >> 3; + job->cipher_mode = IMB_CIPHER_ECB; + job->hash_alg = IMB_AUTH_NULL; + job->cipher_start_src_offset_in_bytes = 0; + job->enc_keys = enc_keys; + job->dec_keys = dec_keys; + + if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = tc->pt; + job->dst = tc->ct; + job->msg_len_to_cipher_in_bytes = tc->pt_len; + tc->ct_len = tc->pt_len; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + const int err = imb_get_errno(mb_mgr); + const char *err_str = imb_get_strerror(err); + + fprintf(stderr, "Invalid encrypt job: %s\n", err_str); + return ACVP_CRYPTO_MODULE_FAIL; + } + } else /* DECRYPT */ { + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = tc->ct; + job->dst = tc->pt; + job->msg_len_to_cipher_in_bytes = tc->ct_len; + tc->pt_len = tc->ct_len; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + const int err = imb_get_errno(mb_mgr); + const char *err_str = imb_get_strerror(err); + + fprintf(stderr, "Invalid decrypt job: %s\n", err_str); + return ACVP_CRYPTO_MODULE_FAIL; + } + } + + return ACVP_SUCCESS; } static int aes_gcm_handler(ACVP_TEST_CASE *test_case) @@ -155,13 +237,13 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) struct gcm_context_data ctx; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.symmetric; if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } switch (tc->key_len) { @@ -176,7 +258,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } if (direct_api == 1) { @@ -200,7 +282,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } else { job = IMB_GET_NEXT_JOB(mb_mgr); @@ -240,7 +322,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } } else /* DECRYPT */ { @@ -267,7 +349,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } if (memcmp(res_tag, tc->tag, tc->tag_len) != 0) { @@ -276,12 +358,12 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) res_tag, tc->tag_len); hexdump(stdout, "reference tag: ", tc->tag, tc->tag_len); - fprintf(stderr, "Invalid tag\n"); + fprintf(stderr, "Tag mismatch\n"); } - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int aes_gmac_handler(ACVP_TEST_CASE *test_case) @@ -296,13 +378,13 @@ static int aes_gmac_handler(ACVP_TEST_CASE *test_case) IMB_HASH_ALG hash_mode; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.symmetric; if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } switch (tc->key_len) { @@ -320,7 +402,7 @@ static int aes_gmac_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } if (direct_api == 1) { @@ -340,7 +422,7 @@ static int aes_gmac_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } else { job = IMB_GET_NEXT_JOB(mb_mgr); @@ -373,7 +455,7 @@ static int aes_gmac_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } } else /* DECRYPT */ { @@ -395,7 +477,7 @@ static int aes_gmac_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } if (memcmp(res_tag, tc->tag, tc->tag_len) != 0) { @@ -404,12 +486,12 @@ static int aes_gmac_handler(ACVP_TEST_CASE *test_case) res_tag, tc->tag_len); hexdump(stdout, "reference tag: ", tc->tag, tc->tag_len); - fprintf(stderr, "Invalid tag\n"); + fprintf(stderr, "Tag mismatch\n"); } - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int aes_ctr_handler(ACVP_TEST_CASE *test_case) @@ -420,14 +502,14 @@ static int aes_ctr_handler(ACVP_TEST_CASE *test_case) DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.symmetric; if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { fprintf(stderr, "Unsupported direction\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } switch (tc->key_len) { @@ -442,7 +524,7 @@ static int aes_ctr_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } job = IMB_GET_NEXT_JOB(mb_mgr); @@ -469,7 +551,7 @@ static int aes_ctr_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } else /* DECRYPT */ { job->cipher_direction = IMB_DIR_DECRYPT; @@ -484,10 +566,133 @@ static int aes_ctr_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } - return EXIT_SUCCESS; + return ACVP_SUCCESS; +} + +static int tdes_cbc_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_SYM_CIPHER_TC *tc; + IMB_JOB *job = NULL; + static DECLARE_ALIGNED(uint64_t keys1[IMB_DES_KEY_SCHED_SIZE / sizeof(uint64_t)], 16); + static DECLARE_ALIGNED(uint64_t keys2[IMB_DES_KEY_SCHED_SIZE / sizeof(uint64_t)], 16); + static DECLARE_ALIGNED(uint64_t keys3[IMB_DES_KEY_SCHED_SIZE / sizeof(uint64_t)], 16); + static const void *ks_ptr[3]; + static uint8_t next_iv[8]; + + if (test_case == NULL) + return ACVP_CRYPTO_MODULE_FAIL; + + tc = test_case->tc.symmetric; + + if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && + tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { + fprintf(stderr, "Unsupported direction\n"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + if (tc->keyingOption != 1) { + fprintf(stderr, "Unsupported keyingOption\n"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + /* + * Only 3 key DES supported + */ + if (tc->key_len != 192) { + fprintf(stderr, "Unsupported DES key length\n"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + /* + * Only TDES CBC supported + */ + const ACVP_SUB_TDES alg = acvp_get_tdes_alg(tc->cipher); + + if (alg == 0) { + fprintf(stderr, "Invalid cipher value"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + if (alg != ACVP_SUB_TDES_CBC) { + fprintf(stderr, "Error: Unsupported DES mode requested by ACVP server\n"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + /* Create key schedules */ + if (tc->test_type != ACVP_SYM_TEST_TYPE_MCT || + (tc->test_type == ACVP_SYM_TEST_TYPE_MCT && tc->mct_index == 0)) { + /* + * Always create key schedules unless this is continuation of + * Monte Carlo inner loop. + * Not creating key schedules every time in MCT test + * improves performance. + */ + IMB_DES_KEYSCHED(mb_mgr, keys1, &tc->key[0]); + IMB_DES_KEYSCHED(mb_mgr, keys2, &tc->key[8]); + IMB_DES_KEYSCHED(mb_mgr, keys3, &tc->key[16]); + ks_ptr[0] = keys1; + ks_ptr[1] = keys2; + ks_ptr[2] = keys3; + } + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = 192 / 8; + job->cipher_mode = IMB_CIPHER_DES3; + job->hash_alg = IMB_AUTH_NULL; + + job->iv = tc->iv; + + if (tc->test_type == ACVP_SYM_TEST_TYPE_MCT && + tc->direction == ACVP_SYM_CIPH_DIR_DECRYPT && + tc->mct_index != 0) + job->iv = next_iv; + + job->iv_len_in_bytes = tc->iv_len; + job->cipher_start_src_offset_in_bytes = 0; + job->enc_keys = ks_ptr; + job->dec_keys = ks_ptr; + + if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = tc->pt; + job->dst = tc->ct; + job->msg_len_to_cipher_in_bytes = tc->pt_len; + tc->ct_len = tc->pt_len; + } else /* DECRYPT */ { + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = tc->ct; + job->dst = tc->pt; + job->msg_len_to_cipher_in_bytes = tc->ct_len; + tc->pt_len = tc->ct_len; + } + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return ACVP_CRYPTO_MODULE_FAIL; + } + + /* + * If Monte Carlo test: + * encrypt/decrypt - set IV for the next outer iteration + * decrypt - copy the ciphertext as IV for the next inner iteration + */ + if (tc->test_type == ACVP_SYM_TEST_TYPE_MCT) { + if (tc->mct_index == ACVP_DES_MCT_INNER - 1) + memcpy(tc->iv_ret_after, tc->ct, 8); + + if (tc->direction == ACVP_SYM_CIPH_DIR_DECRYPT) + memcpy(next_iv, tc->ct, 8); + } + + return ACVP_SUCCESS; } static int aes_ccm_handler(ACVP_TEST_CASE *test_case) @@ -499,14 +704,14 @@ static int aes_ccm_handler(ACVP_TEST_CASE *test_case) uint8_t res_tag[MAX_TAG_LENGTH] = {0}; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.symmetric; if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { fprintf(stderr, "Unsupported direction\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } switch (tc->key_len) { @@ -521,7 +726,7 @@ static int aes_ccm_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } job = IMB_GET_NEXT_JOB(mb_mgr); @@ -565,7 +770,7 @@ static int aes_ccm_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } if (tc->direction == ACVP_SYM_CIPH_DIR_DECRYPT) { @@ -578,12 +783,12 @@ static int aes_ccm_handler(ACVP_TEST_CASE *test_case) res_tag, tc->tag_len); hexdump(stdout, "reference tag: ", ref_tag, tc->tag_len); - fprintf(stderr, "Invalid tag\n"); + fprintf(stderr, "Tag mismatch\n"); } - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int aes_cmac_handler(ACVP_TEST_CASE *test_case) @@ -596,7 +801,7 @@ static int aes_cmac_handler(ACVP_TEST_CASE *test_case) uint8_t res_tag[MAX_TAG_LENGTH] = {0}; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.cmac; @@ -611,7 +816,7 @@ static int aes_cmac_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } job = IMB_GET_NEXT_JOB(mb_mgr); @@ -642,7 +847,7 @@ static int aes_cmac_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } if (tc->verify == 1) { @@ -652,52 +857,29 @@ static int aes_cmac_handler(ACVP_TEST_CASE *test_case) res_tag, (tc->mac_len)); hexdump(stdout, "reference tag: ", tc->mac, tc->mac_len); - fprintf(stderr, "Invalid tag\n"); + fprintf(stderr, "Tag mismatch\n"); } tc->ver_disposition = ACVP_TEST_DISPOSITION_FAIL; } else tc->ver_disposition = ACVP_TEST_DISPOSITION_PASS; } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int hmac_sha1_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; - uint32_t i = 0; DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); - uint8_t key[IMB_SHA1_BLOCK_SIZE]; - uint8_t buf[IMB_SHA1_BLOCK_SIZE]; - uint32_t key_len = 0; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hmac; - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (tc->key_len <= IMB_SHA1_BLOCK_SIZE) { - memcpy(key, tc->key, tc->key_len); - key_len = (uint32_t) tc->key_len; - } else { - IMB_SHA1(mb_mgr, tc->key, tc->key_len, key); - key_len = IMB_SHA1_DIGEST_SIZE_IN_BYTES; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_1, + tc->key, tc->key_len, ipad_hash, opad_hash); job = IMB_GET_NEXT_JOB(mb_mgr); job->key_len_in_bytes = tc->key_len; @@ -722,48 +904,25 @@ static int hmac_sha1_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int hmac_sha256_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; - uint32_t i = 0; DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); - uint8_t key[IMB_SHA_256_BLOCK_SIZE]; - uint8_t buf[IMB_SHA_256_BLOCK_SIZE]; - uint32_t key_len = 0; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hmac; - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (tc->key_len <= IMB_SHA_256_BLOCK_SIZE) { - memcpy(key, tc->key, tc->key_len); - key_len = (uint32_t) tc->key_len; - } else { - IMB_SHA256(mb_mgr, tc->key, tc->key_len, key); - key_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_256, + tc->key, tc->key_len, ipad_hash, opad_hash); job = IMB_GET_NEXT_JOB(mb_mgr); job->key_len_in_bytes = tc->key_len; @@ -788,48 +947,25 @@ static int hmac_sha256_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int hmac_sha224_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; - uint32_t i = 0; DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA224_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA224_DIGEST_SIZE_IN_BYTES], 16); - uint8_t key[IMB_SHA_256_BLOCK_SIZE]; - uint8_t buf[IMB_SHA_256_BLOCK_SIZE]; - uint32_t key_len = 0; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hmac; - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (tc->key_len <= IMB_SHA_256_BLOCK_SIZE) { - memcpy(key, tc->key, tc->key_len); - key_len = (uint32_t) tc->key_len; - } else { - IMB_SHA224(mb_mgr, tc->key, tc->key_len, key); - key_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_224, + tc->key, tc->key_len, ipad_hash, opad_hash); job = IMB_GET_NEXT_JOB(mb_mgr); job->key_len_in_bytes = tc->key_len; @@ -854,48 +990,25 @@ static int hmac_sha224_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int hmac_sha384_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; - uint32_t i = 0; DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - uint8_t key[IMB_SHA_384_BLOCK_SIZE]; - uint8_t buf[IMB_SHA_384_BLOCK_SIZE]; - uint32_t key_len = 0; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hmac; - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (tc->key_len <= IMB_SHA_384_BLOCK_SIZE) { - memcpy(key, tc->key, tc->key_len); - key_len = (uint32_t) tc->key_len; - } else { - IMB_SHA384(mb_mgr, tc->key, tc->key_len, key); - key_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_384, + tc->key, tc->key_len, ipad_hash, opad_hash); job = IMB_GET_NEXT_JOB(mb_mgr); job->key_len_in_bytes = tc->key_len; @@ -920,48 +1033,25 @@ static int hmac_sha384_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int hmac_sha512_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; - uint32_t i = 0; DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - uint8_t key[IMB_SHA_512_BLOCK_SIZE]; - uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; - uint32_t key_len = 0; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hmac; - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (tc->key_len <= IMB_SHA_512_BLOCK_SIZE) { - memcpy(key, tc->key, tc->key_len); - key_len = (uint32_t) tc->key_len; - } else { - IMB_SHA512(mb_mgr, tc->key, tc->key_len, key); - key_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_512, + tc->key, tc->key_len, ipad_hash, opad_hash); job = IMB_GET_NEXT_JOB(mb_mgr); job->key_len_in_bytes = tc->key_len; @@ -986,9 +1076,9 @@ static int hmac_sha512_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int sha1_handler(ACVP_TEST_CASE *test_case) @@ -999,7 +1089,7 @@ static int sha1_handler(ACVP_TEST_CASE *test_case) uint8_t *m; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hash; @@ -1009,7 +1099,7 @@ static int sha1_handler(ACVP_TEST_CASE *test_case) if (m == NULL) { printf("Can't allocate buffer memory\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } memcpy(m, tc->m1, tc->msg_len); memcpy(m + tc->msg_len, tc->m2, tc->msg_len); @@ -1040,13 +1130,13 @@ static int sha1_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) free(m); tc->md_len = IMB_SHA1_DIGEST_SIZE_IN_BYTES; - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int sha2_224_handler(ACVP_TEST_CASE *test_case) @@ -1057,7 +1147,7 @@ static int sha2_224_handler(ACVP_TEST_CASE *test_case) uint8_t *m; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hash; @@ -1067,7 +1157,7 @@ static int sha2_224_handler(ACVP_TEST_CASE *test_case) if (m == NULL) { printf("Can't allocate buffer memory\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } memcpy(m, tc->m1, tc->msg_len); memcpy(m + tc->msg_len, tc->m2, tc->msg_len); @@ -1098,13 +1188,13 @@ static int sha2_224_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) free(m); tc->md_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int sha2_256_handler(ACVP_TEST_CASE *test_case) @@ -1115,7 +1205,7 @@ static int sha2_256_handler(ACVP_TEST_CASE *test_case) uint8_t *m; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hash; @@ -1125,7 +1215,7 @@ static int sha2_256_handler(ACVP_TEST_CASE *test_case) if (m == NULL) { printf("Can't allocate buffer memory\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } memcpy(m, tc->m1, tc->msg_len); memcpy(m + tc->msg_len, tc->m2, tc->msg_len); @@ -1156,13 +1246,13 @@ static int sha2_256_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) free(m); tc->md_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int sha2_384_handler(ACVP_TEST_CASE *test_case) @@ -1173,7 +1263,7 @@ static int sha2_384_handler(ACVP_TEST_CASE *test_case) uint8_t *m; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hash; @@ -1183,7 +1273,7 @@ static int sha2_384_handler(ACVP_TEST_CASE *test_case) if (m == NULL) { printf("Can't allocate buffer memory\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } memcpy(m, tc->m1, tc->msg_len); memcpy(m + tc->msg_len, tc->m2, tc->msg_len); @@ -1214,13 +1304,13 @@ static int sha2_384_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) free(m); tc->md_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static int sha2_512_handler(ACVP_TEST_CASE *test_case) @@ -1231,7 +1321,7 @@ static int sha2_512_handler(ACVP_TEST_CASE *test_case) uint8_t *m; if (test_case == NULL) - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; tc = test_case->tc.hash; @@ -1241,7 +1331,7 @@ static int sha2_512_handler(ACVP_TEST_CASE *test_case) if (m == NULL) { printf("Can't allocate buffer memory\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } memcpy(m, tc->m1, tc->msg_len); memcpy(m + tc->msg_len, tc->m2, tc->msg_len); @@ -1272,13 +1362,13 @@ static int sha2_512_handler(ACVP_TEST_CASE *test_case) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { fprintf(stderr, "Invalid job\n"); - return EXIT_FAILURE; + return ACVP_CRYPTO_MODULE_FAIL; } } if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) free(m); tc->md_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; - return EXIT_SUCCESS; + return ACVP_SUCCESS; } static void usage(const char *app_name) @@ -1338,15 +1428,15 @@ int main(int argc, char **argv) "Missing argument for --arch\n"); goto exit; } - if (strcmp(argv[i], "SSE") == 0) + if (strcmp(argv[i + 1], "SSE") == 0) test_arch = IMB_ARCH_SSE; - else if (strcmp(argv[i], "AVX") == 0) + else if (strcmp(argv[i + 1], "AVX") == 0) test_arch = IMB_ARCH_AVX; - else if (strcmp(argv[i], "AVX2") == 0) + else if (strcmp(argv[i + 1], "AVX2") == 0) test_arch = IMB_ARCH_AVX2; - else if (strcmp(argv[i], "AVX512") == 0) + else if (strcmp(argv[i + 1], "AVX512") == 0) test_arch = IMB_ARCH_AVX512; - else if (strcmp(argv[i], "NO-AESNI") == 0) + else if (strcmp(argv[i + 1], "NO-AESNI") == 0) test_arch = IMB_ARCH_NOAESNI; else { fprintf(stderr, "Unsupported architecture\n"); @@ -1376,8 +1466,14 @@ int main(int argc, char **argv) goto exit; } + printf("ACVP library: %s\n", acvp_version()); + /* Create test session and enable supported algorithms */ - acvp_ret = acvp_create_test_session(&ctx, logger, ACVP_LOG_LVL_INFO); + if (verbose) + acvp_ret = acvp_create_test_session(&ctx, logger, ACVP_LOG_LVL_VERBOSE); + else + acvp_ret = acvp_create_test_session(&ctx, logger, ACVP_LOG_LVL_INFO); + if (acvp_ret != ACVP_SUCCESS) goto exit; @@ -1389,13 +1485,22 @@ int main(int argc, char **argv) &aes_cbc_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_ECB, + &aes_ecb_handler) != ACVP_SUCCESS) + goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CTR, &aes_ctr_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_GMAC, &aes_gmac_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_TDES_CBC, + &tdes_cbc_handler) != ACVP_SUCCESS) + goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CCM, &aes_ccm_handler) != ACVP_SUCCESS) goto exit; diff --git a/test/cmac_test.c b/test/cmac_test.c deleted file mode 100644 index 6897d16d4ff0e3f6a496b4ed4c84983360c51d24..0000000000000000000000000000000000000000 --- a/test/cmac_test.c +++ /dev/null @@ -1,1515 +0,0 @@ -/***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include -#include -#include - -#include -#include "gcm_ctr_vectors_test.h" -#include "utils.h" - -enum cmac_type { - CMAC_128 = 0, - CMAC_128_BITLEN, - CMAC_256, -}; - -int cmac_test(struct IMB_MGR *mb_mgr); - -/* - * Test vectors from https://tools.ietf.org/html/rfc4493 - */ - -/* - * Subkey Generation - * K 2b7e1516 28aed2a6 abf71588 09cf4f3c - * AES-128(key,0) 7df76b0c 1ab899b3 3e42f047 b91b546f - * K1 fbeed618 35713366 7c85e08f 7236a8de - * K2 f7ddac30 6ae266cc f90bc11e e46d513b - */ -static const uint8_t key[16] = { - 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, - 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c -}; -static const uint8_t sub_key1[16] = { - 0xfb, 0xee, 0xd6, 0x18, 0x35, 0x71, 0x33, 0x66, - 0x7c, 0x85, 0xe0, 0x8f, 0x72, 0x36, 0xa8, 0xde -}; -static const uint8_t sub_key2[16] = { - 0xf7, 0xdd, 0xac, 0x30, 0x6a, 0xe2, 0x66, 0xcc, - 0xf9, 0x0b, 0xc1, 0x1e, 0xe4, 0x6d, 0x51, 0x3b -}; - -/* - * Example 1: len = 0 - * M - * AES-CMAC bb1d6929 e9593728 7fa37d12 9b756746 - */ -static const uint8_t T_1[16] = { - 0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28, - 0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46 -}; - -/* - * Example 2: len = 16 - * M 6bc1bee2 2e409f96 e93d7e11 7393172a - * AES-CMAC 070a16b4 6b4d4144 f79bdd9d d04a287c - */ -static const uint8_t T_2[16] = { - 0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, - 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c -}; - -/* - * Example 3: len = 40 - * M 6bc1bee2 2e409f96 e93d7e11 7393172a - * ae2d8a57 1e03ac9c 9eb76fac 45af8e51 - * 30c81c46 a35ce411 - * AES-CMAC dfa66747 de9ae630 30ca3261 1497c827 - */ -static const uint8_t T_3[16] = { - 0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30, - 0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27 -}; - -/* - * Example 4: len = 64 - * M 6bc1bee2 2e409f96 e93d7e11 7393172a - * ae2d8a57 1e03ac9c 9eb76fac 45af8e51 - * 30c81c46 a35ce411 e5fbc119 1a0a52ef - * f69f2445 df4f9b17 ad2b417b e66c3710 - * AES-CMAC 51f0bebf 7e3b9d92 fc497417 79363cfe - */ -static const uint8_t T_4[16] = { - 0x51, 0xf0, 0xbe, 0xbf, 0x7e, 0x3b, 0x9d, 0x92, - 0xfc, 0x49, 0x74, 0x17, 0x79, 0x36, 0x3c, 0xfe -}; - -/* - * Custom Vector - * - * Example 5: len = 8 - * M 6bc1bee2 2e409f96 - * AES-CMAC dc87cdcf 77a2f182 9e012c4d 31af2f8b - */ -static const uint8_t T_5[16] = { - 0xdc, 0x87, 0xcd, 0xcf, 0x77, 0xa2, 0xf1, 0x82, - 0x9e, 0x01, 0x2c, 0x4d, 0x31, 0xaf, 0x2f, 0x8b -}; - -static const uint8_t M[64] = { - 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, - 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, - 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, - 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, - 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, - 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, - 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, - 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 1 - * - * CMAC(K,M): - * K = (hex) 2bd6459f 82c5b300 952c4910 4881ff48 - * Mlen = 122 (bits) - * M = (hex) 38a6f056 c0000000 33323462 63393840 - * - * Subkey generation: - * K1 = (hex) dc84c270 b5bf83f9 6f90be18 8d3f6418 - * K2 = (hex) b90984e1 6b7f07f2 df217c31 1a7ec8b7 - * - * MAC generation: - * C1 = (hex) 118c6eb8 b775144b 0b831110 54c96eb6 - * MACT = (hex) 118c6eb8 - */ -static const uint8_t EIA2_128_K_1[16] = { - 0x2b, 0xd6, 0x45, 0x9f, 0x82, 0xc5, 0xb3, 0x00, - 0x95, 0x2c, 0x49, 0x10, 0x48, 0x81, 0xff, 0x48 -}; - -static const uint8_t EIA2_128_SK1_1[16] = { - 0xdc, 0x84, 0xc2, 0x70, 0xb5, 0xbf, 0x83, 0xf9, - 0x6f, 0x90, 0xbe, 0x18, 0x8d, 0x3f, 0x64, 0x18 -}; - -static const uint8_t EIA2_128_SK2_1[16] = { - 0xb9, 0x09, 0x84, 0xe1, 0x6b, 0x7f, 0x07, 0xf2, - 0xdf, 0x21, 0x7c, 0x31, 0x1a, 0x7e, 0xc8, 0xb7 -}; - -static const uint8_t EIA2_128_T_1[4] = { - 0x11, 0x8c, 0x6e, 0xb8 -}; - -static const uint8_t EIA2_128_M_1[16] = { - 0x38, 0xa6, 0xf0, 0x56, 0xc0, 0x00, 0x00, 0x00, - 0x33, 0x32, 0x34, 0x62, 0x63, 0x39, 0x38, 0x40 /* 0x40 = 0100 0000 */ -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 2 - * - * CMAC(K, M): - * K = d3c5d592 327fb11c 4035c668 0af8c6d1 - * Mlen = 128 - * M = 398a59b4 d4000000 484583d5 afe082ae - * - * Subkey Generation: - * L = 9b71f299 132915d3 605211b5 e5df8632 - * K1 = 36e3e532 26522ba6 c0a4236b cbbf0ce3 - * K2 = 6dc7ca64 4ca4574d 814846d7 977e19c6 - * - * MAC generation: - * C1 = b93787e6 493ff113 ad73d3e0 1e826d73 - * MACT = b93787e6 - */ -static const uint8_t EIA2_128_K_2[16] = { - 0xd3, 0xc5, 0xd5, 0x92, 0x32, 0x7f, 0xb1, 0x1c, - 0x40, 0x35, 0xc6, 0x68, 0x0a, 0xf8, 0xc6, 0xd1 -}; - -static const uint8_t EIA2_128_SK1_2[16] = { - 0x36, 0xe3, 0xe5, 0x32, 0x26, 0x52, 0x2b, 0xa6, - 0xc0, 0xa4, 0x23, 0x6b, 0xcb, 0xbf, 0x0c, 0xe3 -}; - -static const uint8_t EIA2_128_SK2_2[16] = { - 0x6d, 0xc7, 0xca, 0x64, 0x4c, 0xa4, 0x57, 0x4d, - 0x81, 0x48, 0x46, 0xd7, 0x97, 0x7e, 0x19, 0xc6 -}; - -static const uint8_t EIA2_128_T_2[4] = { - 0xb9, 0x37, 0x87, 0xe6 -}; - -static const uint8_t EIA2_128_M_2[16] = { - 0x39, 0x8a, 0x59, 0xb4, 0xd4, 0x00, 0x00, 0x00, - 0x48, 0x45, 0x83, 0xd5, 0xaf, 0xe0, 0x82, 0xae -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 3 - * - * CMAC(K, M): - * K = 7e5e9443 1e11d738 28d739cc 6ced4573 - * Mlen = 318 - * M = 36af6144 c4000000 b3d3c917 0a4e1632 f60f8610 13d22d84 b726b6a2 - * 78d802d1 eeaf1321 ba5929dc - * - * Subkey Generation: - * L = d78b4628 35781e79 d2255f8d 309a60ef - * K1 = af168c50 6af03cf3 a44abf1a 6134c159 - * K2 = 5e2d18a0 d5e079e7 48957e34 c2698235 - * - * MAC generation: - * C3 = 1f60b01d e05aa666 3bda32c6 1771e70b - * MACT = 1f60b01d - */ -static const uint8_t EIA2_128_K_3[16] = { - 0x7e, 0x5e, 0x94, 0x43, 0x1e, 0x11, 0xd7, 0x38, - 0x28, 0xd7, 0x39, 0xcc, 0x6c, 0xed, 0x45, 0x73 -}; - -static const uint8_t EIA2_128_SK1_3[16] = { - 0xaf, 0x16, 0x8c, 0x50, 0x6a, 0xf0, 0x3c, 0xf3, - 0xa4, 0x4a, 0xbf, 0x1a, 0x61, 0x34, 0xc1, 0x59 -}; - -static const uint8_t EIA2_128_SK2_3[16] = { - 0x5e, 0x2d, 0x18, 0xa0, 0xd5, 0xe0, 0x79, 0xe7, - 0x48, 0x95, 0x7e, 0x34, 0xc2, 0x69, 0x82, 0x35 -}; - -static const uint8_t EIA2_128_T_3[4] = { - 0x1f, 0x60, 0xb0, 0x1d -}; - -static const uint8_t EIA2_128_M_3[40] = { - 0x36, 0xaf, 0x61, 0x44, 0xc4, 0x00, 0x00, 0x00, - 0xb3, 0xd3, 0xc9, 0x17, 0x0a, 0x4e, 0x16, 0x32, - 0xf6, 0x0f, 0x86, 0x10, 0x13, 0xd2, 0x2d, 0x84, - 0xb7, 0x26, 0xb6, 0xa2, 0x78, 0xd8, 0x02, 0xd1, - 0xee, 0xaf, 0x13, 0x21, 0xba, 0x59, 0x29, 0xdc -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 4 - * - * CMAC(K, M): - * K = d3419be8 21087acd 02123a92 48033359 - * Mlen = 575 - * M = c7590ea9 b8000000 bbb05703 8809496b - * cff86d6f bc8ce5b1 35a06b16 6054f2d5 - * 65be8ace 75dc851e 0bcdd8f0 7141c495 - * 872fb5d8 c0c66a8b 6da55666 3e4e4612 - * 05d84580 bee5bc7e - * - * Subkey Generation: - * L = 054dd008 2d9ecd21 a3f32b0a a7369be4 - * K1 = 0a9ba010 5b3d9a43 47e65615 4e6d37c8 - * K2 = 15374020 b67b3486 8fccac2a 9cda6f90 - * - * MAC generation: - * C5 = 6846a2f0 a0b6be7a 4fb26a15 7e914c53 - * MACT = 6846a2f0 - */ -static const uint8_t EIA2_128_K_4[16] = { - 0xd3, 0x41, 0x9b, 0xe8, 0x21, 0x08, 0x7a, 0xcd, - 0x02, 0x12, 0x3a, 0x92, 0x48, 0x03, 0x33, 0x59 -}; - -static const uint8_t EIA2_128_SK1_4[16] = { - 0x0a, 0x9b, 0xa0, 0x10, 0x5b, 0x3d, 0x9a, 0x43, - 0x47, 0xe6, 0x56, 0x15, 0x4e, 0x6d, 0x37, 0xc8 -}; - -static const uint8_t EIA2_128_SK2_4[16] = { - 0x15, 0x37, 0x40, 0x20, 0xb6, 0x7b, 0x34, 0x86, - 0x8f, 0xcc, 0xac, 0x2a, 0x9c, 0xda, 0x6f, 0x90 -}; - -static const uint8_t EIA2_128_T_4[4] = { - 0x68, 0x46, 0xa2, 0xf0 -}; - -static const uint8_t EIA2_128_M_4[72] = { - 0xc7, 0x59, 0x0e, 0xa9, 0xb8, 0x00, 0x00, 0x00, - 0xbb, 0xb0, 0x57, 0x03, 0x88, 0x09, 0x49, 0x6b, - 0xcf, 0xf8, 0x6d, 0x6f, 0xbc, 0x8c, 0xe5, 0xb1, - 0x35, 0xa0, 0x6b, 0x16, 0x60, 0x54, 0xf2, 0xd5, - 0x65, 0xbe, 0x8a, 0xce, 0x75, 0xdc, 0x85, 0x1e, - 0x0b, 0xcd, 0xd8, 0xf0, 0x71, 0x41, 0xc4, 0x95, - 0x87, 0x2f, 0xb5, 0xd8, 0xc0, 0xc6, 0x6a, 0x8b, - 0x6d, 0xa5, 0x56, 0x66, 0x3e, 0x4e, 0x46, 0x12, - 0x05, 0xd8, 0x45, 0x80, 0xbe, 0xe5, 0xbc, 0x7e -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 5 - * - * CMAC(K, M): - * K = 83fd23a2 44a74cf3 58da3019 f1722635 - * Mlen = 832 - * M = 36af6144 7c000000 35c68716 633c66fb - * 750c2668 65d53c11 ea05b1e9 fa49c839 - * 8d48e1ef a5909d39 47902837 f5ae96d5 - * a05bc8d6 1ca8dbef 1b13a4b4 abfe4fb1 - * 006045b6 74bb5472 9304c382 be53a5af - * 05556176 f6eaa2ef 1d05e4b0 83181ee6 - * 74cda5a4 85f74d7a - * - * Subkey Generation: - * L = 9df61c57 3c86acac 704db9d5 b0dea444 - * K1 = 3bec38ae 790d5958 e09b73ab 61bd480f - * K2 = 77d8715c f21ab2b1 c136e756 c37a901e - * - * MAC generation: - * C7 = e657e182 5298f2fa ee2ca1e0 7373bc7e - * MACT = e657e182 - */ -static const uint8_t EIA2_128_K_5[16] = { - 0x83, 0xfd, 0x23, 0xa2, 0x44, 0xa7, 0x4c, 0xf3, - 0x58, 0xda, 0x30, 0x19, 0xf1, 0x72, 0x26, 0x35 -}; - -static const uint8_t EIA2_128_SK1_5[16] = { - 0x3b, 0xec, 0x38, 0xae, 0x79, 0x0d, 0x59, 0x58, - 0xe0, 0x9b, 0x73, 0xab, 0x61, 0xbd, 0x48, 0x0f -}; - -static const uint8_t EIA2_128_SK2_5[16] = { - 0x77, 0xd8, 0x71, 0x5c, 0xf2, 0x1a, 0xb2, 0xb1, - 0xc1, 0x36, 0xe7, 0x56, 0xc3, 0x7a, 0x90, 0x1e -}; - -static const uint8_t EIA2_128_T_5[4] = { - 0xe6, 0x57, 0xe1, 0x82 -}; - -static const uint8_t EIA2_128_M_5[104] = { - 0x36, 0xaf, 0x61, 0x44, 0x7c, 0x00, 0x00, 0x00, - 0x35, 0xc6, 0x87, 0x16, 0x63, 0x3c, 0x66, 0xfb, - 0x75, 0x0c, 0x26, 0x68, 0x65, 0xd5, 0x3c, 0x11, - 0xea, 0x05, 0xb1, 0xe9, 0xfa, 0x49, 0xc8, 0x39, - 0x8d, 0x48, 0xe1, 0xef, 0xa5, 0x90, 0x9d, 0x39, - 0x47, 0x90, 0x28, 0x37, 0xf5, 0xae, 0x96, 0xd5, - 0xa0, 0x5b, 0xc8, 0xd6, 0x1c, 0xa8, 0xdb, 0xef, - 0x1b, 0x13, 0xa4, 0xb4, 0xab, 0xfe, 0x4f, 0xb1, - 0x00, 0x60, 0x45, 0xb6, 0x74, 0xbb, 0x54, 0x72, - 0x93, 0x04, 0xc3, 0x82, 0xbe, 0x53, 0xa5, 0xaf, - 0x05, 0x55, 0x61, 0x76, 0xf6, 0xea, 0xa2, 0xef, - 0x1d, 0x05, 0xe4, 0xb0, 0x83, 0x18, 0x1e, 0xe6, - 0x74, 0xcd, 0xa5, 0xa4, 0x85, 0xf7, 0x4d, 0x7a -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 6 - * - * CMAC(K, M): - * K = 6832a65c ff447362 1ebdd4ba 26a921fe - * Mlen = 447 - * M = 36af6144 c0000000 d3c53839 62682071 - * 77656676 20323837 63624098 1ba6824c - * 1bfb1ab4 85472029 b71d808c e33e2cc3 - * c0b5fc1f 3de8a6dc - * - * Subkey Generation: - * L = e50123c3 87e13fd6 8d8bf0d0 a4581685 - * K1 = ca024787 0fc27fad 1b17e1a1 48b02d8d - * K2 = 94048f0e 1f84ff5a 362fc342 91605b9d - * - * MAC generation: - * C4 = f0668c1e 4197300b 1243f834 25d06c25 - * MACT = f0668c1e - */ -static const uint8_t EIA2_128_K_6[16] = { - 0x68, 0x32, 0xa6, 0x5c, 0xff, 0x44, 0x73, 0x62, - 0x1e, 0xbd, 0xd4, 0xba, 0x26, 0xa9, 0x21, 0xfe -}; - -static const uint8_t EIA2_128_SK1_6[16] = { - 0xca, 0x02, 0x47, 0x87, 0x0f, 0xc2, 0x7f, 0xad, - 0x1b, 0x17, 0xe1, 0xa1, 0x48, 0xb0, 0x2d, 0x8d -}; - -static const uint8_t EIA2_128_SK2_6[16] = { - 0x94, 0x04, 0x8f, 0x0e, 0x1f, 0x84, 0xff, 0x5a, - 0x36, 0x2f, 0xc3, 0x42, 0x91, 0x60, 0x5b, 0x9d -}; - -static const uint8_t EIA2_128_T_6[4] = { - 0xf0, 0x66, 0x8c, 0x1e -}; - -static const uint8_t EIA2_128_M_6[56] = { - 0x36, 0xaf, 0x61, 0x44, 0xc0, 0x00, 0x00, 0x00, - 0xd3, 0xc5, 0x38, 0x39, 0x62, 0x68, 0x20, 0x71, - 0x77, 0x65, 0x66, 0x76, 0x20, 0x32, 0x38, 0x37, - 0x63, 0x62, 0x40, 0x98, 0x1b, 0xa6, 0x82, 0x4c, - 0x1b, 0xfb, 0x1a, 0xb4, 0x85, 0x47, 0x20, 0x29, - 0xb7, 0x1d, 0x80, 0x8c, 0xe3, 0x3e, 0x2c, 0xc3, - 0xc0, 0xb5, 0xfc, 0x1f, 0x3d, 0xe8, 0xa6, 0xdc -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 7 - * - * CMAC(K, M): - * K = 5d0a80d8 134ae196 77824b67 1e838af4 - * Mlen = 2622 - * M = 7827fab2 2c000000 70dedf2d c42c5cbd - * 3a96f8a0 b11418b3 608d5733 604a2cd3 - * 6aabc70c e3193bb5 153be2d3 c06dfdb2 - * d16e9c35 7158be6a 41d6b861 e491db3f - * bfeb518e fcf048d7 d5895373 0ff30c9e - * c470ffcd 663dc342 01c36add c0111c35 - * b38afee7 cfdb582e 3731f8b4 baa8d1a8 - * 9c06e811 99a97162 27be344e fcb436dd - * d0f096c0 64c3b5e2 c399993f c77394f9 - * e09720a8 11850ef2 3b2ee05d 9e617360 - * 9d86e1c0 c18ea51a 012a00bb 413b9cb8 - * 188a703c d6bae31c c67b34b1 b00019e6 - * a2b2a690 f02671fe 7c9ef8de c0094e53 - * 3763478d 58d2c5f5 b827a014 8c5948a9 - * 6931acf8 4f465a64 e62ce740 07e991e3 - * 7ea823fa 0fb21923 b79905b7 33b631e6 - * c7d6860a 3831ac35 1a9c730c 52ff72d9 - * d308eedb ab21fde1 43a0ea17 e23edc1f - * 74cbb363 8a2033aa a15464ea a733385d - * bbeb6fd7 3509b857 e6a419dc a1d8907a - * f977fbac 4dfa35ec - * - * Subkey Generation: - * L = 9832e229 fbb93970 bcf7b282 3ee4fe5d - * K1 = 3065c453 f77272e1 79ef6504 7dc9fc3d - * K2 = 60cb88a7 eee4e5c2 f3deca08 fb93f87a - * - * MAC generation: - * C21 = f4cc8fa3 59e6e2e7 6e09c45d 6ea5e0de - * MACT = f4cc8fa3 - */ -static const uint8_t EIA2_128_K_7[16] = { - 0x5d, 0x0a, 0x80, 0xd8, 0x13, 0x4a, 0xe1, 0x96, - 0x77, 0x82, 0x4b, 0x67, 0x1e, 0x83, 0x8a, 0xf4 -}; - -static const uint8_t EIA2_128_SK1_7[16] = { - 0x30, 0x65, 0xc4, 0x53, 0xf7, 0x72, 0x72, 0xe1, - 0x79, 0xef, 0x65, 0x04, 0x7d, 0xc9, 0xfc, 0x3d -}; - -static const uint8_t EIA2_128_SK2_7[16] = { - 0x60, 0xcb, 0x88, 0xa7, 0xee, 0xe4, 0xe5, 0xc2, - 0xf3, 0xde, 0xca, 0x08, 0xfb, 0x93, 0xf8, 0x7a -}; - -static const uint8_t EIA2_128_T_7[4] = { - 0xf4, 0xcc, 0x8f, 0xa3 -}; - -static const uint8_t EIA2_128_M_7[328] = { - 0x78, 0x27, 0xfa, 0xb2, 0x2c, 0x00, 0x00, 0x00, - 0x70, 0xde, 0xdf, 0x2d, 0xc4, 0x2c, 0x5c, 0xbd, - 0x3a, 0x96, 0xf8, 0xa0, 0xb1, 0x14, 0x18, 0xb3, - 0x60, 0x8d, 0x57, 0x33, 0x60, 0x4a, 0x2c, 0xd3, - 0x6a, 0xab, 0xc7, 0x0c, 0xe3, 0x19, 0x3b, 0xb5, - 0x15, 0x3b, 0xe2, 0xd3, 0xc0, 0x6d, 0xfd, 0xb2, - 0xd1, 0x6e, 0x9c, 0x35, 0x71, 0x58, 0xbe, 0x6a, - 0x41, 0xd6, 0xb8, 0x61, 0xe4, 0x91, 0xdb, 0x3f, - 0xbf, 0xeb, 0x51, 0x8e, 0xfc, 0xf0, 0x48, 0xd7, - 0xd5, 0x89, 0x53, 0x73, 0x0f, 0xf3, 0x0c, 0x9e, - 0xc4, 0x70, 0xff, 0xcd, 0x66, 0x3d, 0xc3, 0x42, - 0x01, 0xc3, 0x6a, 0xdd, 0xc0, 0x11, 0x1c, 0x35, - 0xb3, 0x8a, 0xfe, 0xe7, 0xcf, 0xdb, 0x58, 0x2e, - 0x37, 0x31, 0xf8, 0xb4, 0xba, 0xa8, 0xd1, 0xa8, - 0x9c, 0x06, 0xe8, 0x11, 0x99, 0xa9, 0x71, 0x62, - 0x27, 0xbe, 0x34, 0x4e, 0xfc, 0xb4, 0x36, 0xdd, - 0xd0, 0xf0, 0x96, 0xc0, 0x64, 0xc3, 0xb5, 0xe2, - 0xc3, 0x99, 0x99, 0x3f, 0xc7, 0x73, 0x94, 0xf9, - 0xe0, 0x97, 0x20, 0xa8, 0x11, 0x85, 0x0e, 0xf2, - 0x3b, 0x2e, 0xe0, 0x5d, 0x9e, 0x61, 0x73, 0x60, - 0x9d, 0x86, 0xe1, 0xc0, 0xc1, 0x8e, 0xa5, 0x1a, - 0x01, 0x2a, 0x00, 0xbb, 0x41, 0x3b, 0x9c, 0xb8, - 0x18, 0x8a, 0x70, 0x3c, 0xd6, 0xba, 0xe3, 0x1c, - 0xc6, 0x7b, 0x34, 0xb1, 0xb0, 0x00, 0x19, 0xe6, - 0xa2, 0xb2, 0xa6, 0x90, 0xf0, 0x26, 0x71, 0xfe, - 0x7c, 0x9e, 0xf8, 0xde, 0xc0, 0x09, 0x4e, 0x53, - 0x37, 0x63, 0x47, 0x8d, 0x58, 0xd2, 0xc5, 0xf5, - 0xb8, 0x27, 0xa0, 0x14, 0x8c, 0x59, 0x48, 0xa9, - 0x69, 0x31, 0xac, 0xf8, 0x4f, 0x46, 0x5a, 0x64, - 0xe6, 0x2c, 0xe7, 0x40, 0x07, 0xe9, 0x91, 0xe3, - 0x7e, 0xa8, 0x23, 0xfa, 0x0f, 0xb2, 0x19, 0x23, - 0xb7, 0x99, 0x05, 0xb7, 0x33, 0xb6, 0x31, 0xe6, - 0xc7, 0xd6, 0x86, 0x0a, 0x38, 0x31, 0xac, 0x35, - 0x1a, 0x9c, 0x73, 0x0c, 0x52, 0xff, 0x72, 0xd9, - 0xd3, 0x08, 0xee, 0xdb, 0xab, 0x21, 0xfd, 0xe1, - 0x43, 0xa0, 0xea, 0x17, 0xe2, 0x3e, 0xdc, 0x1f, - 0x74, 0xcb, 0xb3, 0x63, 0x8a, 0x20, 0x33, 0xaa, - 0xa1, 0x54, 0x64, 0xea, 0xa7, 0x33, 0x38, 0x5d, - 0xbb, 0xeb, 0x6f, 0xd7, 0x35, 0x09, 0xb8, 0x57, - 0xe6, 0xa4, 0x19, 0xdc, 0xa1, 0xd8, 0x90, 0x7a, - 0xf9, 0x77, 0xfb, 0xac, 0x4d, 0xfa, 0x35, 0xec -}; - -/* - * 3GPP 33.401 C.2.1 Test Case 8 - * - * CMAC(K, M): - * K = b3120ffd b2cf6af4 e73eaf2e f4ebec69 - * Mlen = 16512 - * M = 296f393c 5c000000 00000000 00000000 - * 01010101 01010101 e0958045 f3a0bba4 - * e3968346 f0a3b8a7 c02a018a e6407652 - * 26b987c9 13e6cbf0 83570016 cf83efbc - * 61c08251 3e21561a 427c009d 28c298ef - * ace78ed6 d56c2d45 05ad032e 9c04dc60 - * e73a8169 6da665c6 c48603a5 7b45ab33 - * 221585e6 8ee31691 87fb0239 528632dd - * 656c807e a3248b7b 46d002b2 b5c7458e - * b85b9ce9 5879e034 0859055e 3b0abbc3 - * eace8719 caa80265 c97205d5 dc4bcc90 - * 2fe18396 29ed7132 8a0f0449 f588557e - * 6898860e 042aecd8 4b2404c2 12c9222d - * a5bf8a89 ef679787 0cf50771 a60f66a2 - * ee628536 57addf04 cdde07fa 414e11f1 - * 2b4d81b9 b4e8ac53 8ea30666 688d881f - * 6c348421 992f31b9 4f8806ed 8fccff4c - * 9123b896 42527ad6 13b109bf 75167485 - * f1268bf8 84b4cd23 d29a0934 925703d6 - * 34098f77 67f1be74 91e708a8 bb949a38 - * 73708aef 4a36239e 50cc0823 5cd5ed6b - * be578668 a17b58c1 171d0b90 e813a9e4 - * f58a89d7 19b11042 d6360b1b 0f52deb7 - * 30a58d58 faf46315 954b0a87 26914759 - * 77dc88c0 d733feff 54600a0c c1d0300a - * aaeb9457 2c6e95b0 1ae90de0 4f1dce47 - * f87e8fa7 bebf77e1 dbc20d6b a85cb914 - * 3d518b28 5dfa04b6 98bf0cf7 819f20fa - * 7a288eb0 703d995c 59940c7c 66de57a9 - * b70f8237 9b70e203 1e450fcf d2181326 - * fcd28d88 23baaa80 df6e0f44 35596475 - * 39fd8907 c0ffd9d7 9c130ed8 1c9afd9b - * 7e848c9f ed38443d 5d380e53 fbdb8ac8 - * c3d3f068 76054f12 2461107d e92fea09 - * c6f6923a 188d53af e54a10f6 0e6e9d5a - * 03d996b5 fbc820f8 a637116a 27ad04b4 - * 44a0932d d60fbd12 671c11e1 c0ec73e7 - * 89879faa 3d42c64d 20cd1252 742a3768 - * c25a9015 85888ece e1e612d9 936b403b - * 0775949a 66cdfd99 a29b1345 baa8d9d5 - * 400c9102 4b0a6073 63b013ce 5de9ae86 - * 9d3b8d95 b0570b3c 2d391422 d32450cb - * cfae9665 2286e96d ec1214a9 34652798 - * 0a8192ea c1c39a3a af6f1535 1da6be76 - * 4df89772 ec0407d0 6e4415be fae7c925 - * 80df9bf5 07497c8f 2995160d 4e218daa - * cb02944a bf83340c e8be1686 a960faf9 - * 0e2d90c5 5cc6475b abc3171a 80a36317 - * 4954955d 7101dab1 6ae81791 67e21444 - * b443a9ea aa7c91de 36d118c3 9d389f8d - * d4469a84 6c9a262b f7fa1848 7a79e8de - * 11699e0b 8fdf557c b48719d4 53ba7130 - * 56109b93 a218c896 75ac195f b4fb0663 - * 9b379714 4955b3c9 327d1aec 003d42ec - * d0ea98ab f19ffb4a f3561a67 e77c35bf - * 15c59c24 12da881d b02b1bfb cebfac51 - * 52bc99bc 3f1d15f7 71001b70 29fedb02 - * 8f8b852b c4407eb8 3f891c9c a733254f - * dd1e9edb 56919ce9 fea21c17 4072521c - * 18319a54 b5d4efbe bddf1d8b 69b1cbf2 - * 5f489fcc 98137254 7cf41d00 8ef0bca1 - * 926f934b 735e090b 3b251eb3 3a36f82e - * d9b29cf4 cb944188 fa0e1e38 dd778f7d - * 1c9d987b 28d132df b9731fa4 f4b41693 - * 5be49de3 0516af35 78581f2f 13f561c0 - * 66336194 1eab249a 4bc123f8 d15cd711 - * a956a1bf 20fe6eb7 8aea2373 361da042 - * 6c79a530 c3bb1de0 c99722ef 1fde39ac - * 2b00a0a8 ee7c800a 08bc2264 f89f4eff - * e627ac2f 0531fb55 4f6d21d7 4c590a70 - * adfaa390 bdfbb3d6 8e46215c ab187d23 - * 68d5a71f 5ebec081 cd3b20c0 82dbe4cd - * 2faca287 73795d6b 0c10204b 659a939e - * f29bbe10 88243624 429927a7 eb576dd3 - * a00ea5e0 1af5d475 83b2272c 0c161a80 - * 6521a16f f9b0a722 c0cf26b0 25d5836e - * 2258a4f7 d4773ac8 01e4263b c294f43d - * ef7fa870 3f3a4197 46352588 7652b0b2 - * a4a2a7cf 87f00914 871e2503 9113c7e1 - * 618da340 64b57a43 c463249f b8d05e0f - * 26f4a6d8 4972e7a9 05482414 5f91295c - * dbe39a6f 920facc6 59712b46 a54ba295 - * bbe6a901 54e91b33 985a2bcd 420ad5c6 - * 7ec9ad8e b7ac6864 db272a51 6bc94c28 - * 39b0a816 9a6bf58e 1a0c2ada 8c883b7b - * f497a491 71268ed1 5ddd2969 384e7ff4 - * bf4aab2e c9ecc652 9cf629e2 df0f08a7 - * 7a65afa1 2aa9b505 df8b287e f6cc9149 - * 3d1caa39 076e28ef 1ea028f5 118de61a - * e02bb6ae fc3343a0 50292f19 9f401857 - * b2bead5e 6ee2a1f1 91022f92 78016f04 - * 7791a9d1 8da7d2a6 d27f2e0e 51c2f6ea - * 30e8ac49 a0604f4c 13542e85 b68381b9 - * fdcfa0ce 4b2d3413 54852d36 0245c536 - * b612af71 f3e77c90 95ae2dbd e504b265 - * 733dabfe 10a20fc7 d6d32c21 ccc72b8b - * 3444ae66 3d65922d 17f82caa 2b865cd8 - * 8913d291 a6589902 6ea13284 39723c19 - * 8c36b0c3 c8d085bf af8a320f de334b4a - * 4919b44c 2b95f6e8 ecf73393 f7f0d2a4 - * 0e60b1d4 06526b02 2ddc3318 10b1a5f7 - * c347bd53 ed1f105d 6a0d30ab a477e178 - * 889ab2ec 55d558de ab263020 4336962b - * 4db5b663 b6902b89 e85b31bc 6af50fc5 - * 0accb3fb 9b57b663 29703137 8db47896 - * d7fbaf6c 600add2c 67f936db 037986db - * 856eb49c f2db3f7d a6d23650 e438f188 - * 4041b013 119e4c2a e5af37cc cdfb6866 - * 0738b58b 3c59d1c0 24843747 2aba1f35 - * ca1fb90c d714aa9f 635534f4 9e7c5bba - * 81c2b6b3 6fdee21c a27e347f 793d2ce9 - * 44edb23c 8c9b914b e10335e3 50feb507 - * 0394b7a4 a15c0ca1 20283568 b7bfc254 - * fe838b13 7a2147ce 7c113a3a 4d65499d - * 9e86b87d bcc7f03b bd3a3ab1 aa243ece - * 5ba9bcf2 5f82836c fe473b2d 83e7a720 - * 1cd0b96a 72451e86 3f6c3ba6 64a6d073 - * d1f7b5ed 990865d9 78bd3815 d06094fc - * 9a2aba52 21c22d5a b996389e 3721e3af - * 5f05bedd c2875e0d faeb3902 1ee27a41 - * 187cbb45 ef40c3e7 3bc03989 f9a30d12 - * c54ba7d2 141da8a8 75493e65 776ef35f - * 97debc22 86cc4af9 b4623eee 902f840c - * 52f1b8ad 658939ae f71f3f72 b9ec1de2 - * 1588bd35 484ea444 36343ff9 5ead6ab1 - * d8afb1b2 a303df1b 71e53c4a ea6b2e3e - * 9372be0d 1bc99798 b0ce3cc1 0d2a596d - * 565dba82 f88ce4cf f3b33d5d 24e9c083 - * 1124bf1a d54b7925 32983dd6 c3a8b7d0 - * - * Subkey Generation: - * L = 2c645dcd 72114961 d8b9c864 7aac2c5b - * K1 = 58c8bb9a e42292c3 b17390c8 f55858b6 - * K2 = b1917735 c8452587 62e72191 eab0b16c - * - * MAC generation: - * C129 = ebd5ccb0 b61ca905 29138303 f3377d22 - * MACT = ebd5ccb0 - */ -static const uint8_t EIA2_128_K_8[16] = { - 0xb3, 0x12, 0x0f, 0xfd, 0xb2, 0xcf, 0x6a, 0xf4, - 0xe7, 0x3e, 0xaf, 0x2e, 0xf4, 0xeb, 0xec, 0x69 -}; - -static const uint8_t EIA2_128_SK1_8[16] = { - 0x58, 0xc8, 0xbb, 0x9a, 0xe4, 0x22, 0x92, 0xc3, - 0xb1, 0x73, 0x90, 0xc8, 0xf5, 0x58, 0x58, 0xb6 -}; - -static const uint8_t EIA2_128_SK2_8[16] = { - 0xb1, 0x91, 0x77, 0x35, 0xc8, 0x45, 0x25, 0x87, - 0x62, 0xe7, 0x21, 0x91, 0xea, 0xb0, 0xb1, 0x6c -}; - -static const uint8_t EIA2_128_T_8[4] = { - 0xeb, 0xd5, 0xcc, 0xb0 -}; - -static const uint8_t EIA2_128_M_8[2064] = { - 0x29, 0x6f, 0x39, 0x3c, 0x5c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0xe0, 0x95, 0x80, 0x45, 0xf3, 0xa0, 0xbb, 0xa4, - 0xe3, 0x96, 0x83, 0x46, 0xf0, 0xa3, 0xb8, 0xa7, - 0xc0, 0x2a, 0x01, 0x8a, 0xe6, 0x40, 0x76, 0x52, - 0x26, 0xb9, 0x87, 0xc9, 0x13, 0xe6, 0xcb, 0xf0, - 0x83, 0x57, 0x00, 0x16, 0xcf, 0x83, 0xef, 0xbc, - 0x61, 0xc0, 0x82, 0x51, 0x3e, 0x21, 0x56, 0x1a, - 0x42, 0x7c, 0x00, 0x9d, 0x28, 0xc2, 0x98, 0xef, - 0xac, 0xe7, 0x8e, 0xd6, 0xd5, 0x6c, 0x2d, 0x45, - 0x05, 0xad, 0x03, 0x2e, 0x9c, 0x04, 0xdc, 0x60, - 0xe7, 0x3a, 0x81, 0x69, 0x6d, 0xa6, 0x65, 0xc6, - 0xc4, 0x86, 0x03, 0xa5, 0x7b, 0x45, 0xab, 0x33, - 0x22, 0x15, 0x85, 0xe6, 0x8e, 0xe3, 0x16, 0x91, - 0x87, 0xfb, 0x02, 0x39, 0x52, 0x86, 0x32, 0xdd, - 0x65, 0x6c, 0x80, 0x7e, 0xa3, 0x24, 0x8b, 0x7b, - 0x46, 0xd0, 0x02, 0xb2, 0xb5, 0xc7, 0x45, 0x8e, - 0xb8, 0x5b, 0x9c, 0xe9, 0x58, 0x79, 0xe0, 0x34, - 0x08, 0x59, 0x05, 0x5e, 0x3b, 0x0a, 0xbb, 0xc3, - 0xea, 0xce, 0x87, 0x19, 0xca, 0xa8, 0x02, 0x65, - 0xc9, 0x72, 0x05, 0xd5, 0xdc, 0x4b, 0xcc, 0x90, - 0x2f, 0xe1, 0x83, 0x96, 0x29, 0xed, 0x71, 0x32, - 0x8a, 0x0f, 0x04, 0x49, 0xf5, 0x88, 0x55, 0x7e, - 0x68, 0x98, 0x86, 0x0e, 0x04, 0x2a, 0xec, 0xd8, - 0x4b, 0x24, 0x04, 0xc2, 0x12, 0xc9, 0x22, 0x2d, - 0xa5, 0xbf, 0x8a, 0x89, 0xef, 0x67, 0x97, 0x87, - 0x0c, 0xf5, 0x07, 0x71, 0xa6, 0x0f, 0x66, 0xa2, - 0xee, 0x62, 0x85, 0x36, 0x57, 0xad, 0xdf, 0x04, - 0xcd, 0xde, 0x07, 0xfa, 0x41, 0x4e, 0x11, 0xf1, - 0x2b, 0x4d, 0x81, 0xb9, 0xb4, 0xe8, 0xac, 0x53, - 0x8e, 0xa3, 0x06, 0x66, 0x68, 0x8d, 0x88, 0x1f, - 0x6c, 0x34, 0x84, 0x21, 0x99, 0x2f, 0x31, 0xb9, - 0x4f, 0x88, 0x06, 0xed, 0x8f, 0xcc, 0xff, 0x4c, - 0x91, 0x23, 0xb8, 0x96, 0x42, 0x52, 0x7a, 0xd6, - 0x13, 0xb1, 0x09, 0xbf, 0x75, 0x16, 0x74, 0x85, - 0xf1, 0x26, 0x8b, 0xf8, 0x84, 0xb4, 0xcd, 0x23, - 0xd2, 0x9a, 0x09, 0x34, 0x92, 0x57, 0x03, 0xd6, - 0x34, 0x09, 0x8f, 0x77, 0x67, 0xf1, 0xbe, 0x74, - 0x91, 0xe7, 0x08, 0xa8, 0xbb, 0x94, 0x9a, 0x38, - 0x73, 0x70, 0x8a, 0xef, 0x4a, 0x36, 0x23, 0x9e, - 0x50, 0xcc, 0x08, 0x23, 0x5c, 0xd5, 0xed, 0x6b, - 0xbe, 0x57, 0x86, 0x68, 0xa1, 0x7b, 0x58, 0xc1, - 0x17, 0x1d, 0x0b, 0x90, 0xe8, 0x13, 0xa9, 0xe4, - 0xf5, 0x8a, 0x89, 0xd7, 0x19, 0xb1, 0x10, 0x42, - 0xd6, 0x36, 0x0b, 0x1b, 0x0f, 0x52, 0xde, 0xb7, - 0x30, 0xa5, 0x8d, 0x58, 0xfa, 0xf4, 0x63, 0x15, - 0x95, 0x4b, 0x0a, 0x87, 0x26, 0x91, 0x47, 0x59, - 0x77, 0xdc, 0x88, 0xc0, 0xd7, 0x33, 0xfe, 0xff, - 0x54, 0x60, 0x0a, 0x0c, 0xc1, 0xd0, 0x30, 0x0a, - 0xaa, 0xeb, 0x94, 0x57, 0x2c, 0x6e, 0x95, 0xb0, - 0x1a, 0xe9, 0x0d, 0xe0, 0x4f, 0x1d, 0xce, 0x47, - 0xf8, 0x7e, 0x8f, 0xa7, 0xbe, 0xbf, 0x77, 0xe1, - 0xdb, 0xc2, 0x0d, 0x6b, 0xa8, 0x5c, 0xb9, 0x14, - 0x3d, 0x51, 0x8b, 0x28, 0x5d, 0xfa, 0x04, 0xb6, - 0x98, 0xbf, 0x0c, 0xf7, 0x81, 0x9f, 0x20, 0xfa, - 0x7a, 0x28, 0x8e, 0xb0, 0x70, 0x3d, 0x99, 0x5c, - 0x59, 0x94, 0x0c, 0x7c, 0x66, 0xde, 0x57, 0xa9, - 0xb7, 0x0f, 0x82, 0x37, 0x9b, 0x70, 0xe2, 0x03, - 0x1e, 0x45, 0x0f, 0xcf, 0xd2, 0x18, 0x13, 0x26, - 0xfc, 0xd2, 0x8d, 0x88, 0x23, 0xba, 0xaa, 0x80, - 0xdf, 0x6e, 0x0f, 0x44, 0x35, 0x59, 0x64, 0x75, - 0x39, 0xfd, 0x89, 0x07, 0xc0, 0xff, 0xd9, 0xd7, - 0x9c, 0x13, 0x0e, 0xd8, 0x1c, 0x9a, 0xfd, 0x9b, - 0x7e, 0x84, 0x8c, 0x9f, 0xed, 0x38, 0x44, 0x3d, - 0x5d, 0x38, 0x0e, 0x53, 0xfb, 0xdb, 0x8a, 0xc8, - 0xc3, 0xd3, 0xf0, 0x68, 0x76, 0x05, 0x4f, 0x12, - 0x24, 0x61, 0x10, 0x7d, 0xe9, 0x2f, 0xea, 0x09, - 0xc6, 0xf6, 0x92, 0x3a, 0x18, 0x8d, 0x53, 0xaf, - 0xe5, 0x4a, 0x10, 0xf6, 0x0e, 0x6e, 0x9d, 0x5a, - 0x03, 0xd9, 0x96, 0xb5, 0xfb, 0xc8, 0x20, 0xf8, - 0xa6, 0x37, 0x11, 0x6a, 0x27, 0xad, 0x04, 0xb4, - 0x44, 0xa0, 0x93, 0x2d, 0xd6, 0x0f, 0xbd, 0x12, - 0x67, 0x1c, 0x11, 0xe1, 0xc0, 0xec, 0x73, 0xe7, - 0x89, 0x87, 0x9f, 0xaa, 0x3d, 0x42, 0xc6, 0x4d, - 0x20, 0xcd, 0x12, 0x52, 0x74, 0x2a, 0x37, 0x68, - 0xc2, 0x5a, 0x90, 0x15, 0x85, 0x88, 0x8e, 0xce, - 0xe1, 0xe6, 0x12, 0xd9, 0x93, 0x6b, 0x40, 0x3b, - 0x07, 0x75, 0x94, 0x9a, 0x66, 0xcd, 0xfd, 0x99, - 0xa2, 0x9b, 0x13, 0x45, 0xba, 0xa8, 0xd9, 0xd5, - 0x40, 0x0c, 0x91, 0x02, 0x4b, 0x0a, 0x60, 0x73, - 0x63, 0xb0, 0x13, 0xce, 0x5d, 0xe9, 0xae, 0x86, - 0x9d, 0x3b, 0x8d, 0x95, 0xb0, 0x57, 0x0b, 0x3c, - 0x2d, 0x39, 0x14, 0x22, 0xd3, 0x24, 0x50, 0xcb, - 0xcf, 0xae, 0x96, 0x65, 0x22, 0x86, 0xe9, 0x6d, - 0xec, 0x12, 0x14, 0xa9, 0x34, 0x65, 0x27, 0x98, - 0x0a, 0x81, 0x92, 0xea, 0xc1, 0xc3, 0x9a, 0x3a, - 0xaf, 0x6f, 0x15, 0x35, 0x1d, 0xa6, 0xbe, 0x76, - 0x4d, 0xf8, 0x97, 0x72, 0xec, 0x04, 0x07, 0xd0, - 0x6e, 0x44, 0x15, 0xbe, 0xfa, 0xe7, 0xc9, 0x25, - 0x80, 0xdf, 0x9b, 0xf5, 0x07, 0x49, 0x7c, 0x8f, - 0x29, 0x95, 0x16, 0x0d, 0x4e, 0x21, 0x8d, 0xaa, - 0xcb, 0x02, 0x94, 0x4a, 0xbf, 0x83, 0x34, 0x0c, - 0xe8, 0xbe, 0x16, 0x86, 0xa9, 0x60, 0xfa, 0xf9, - 0x0e, 0x2d, 0x90, 0xc5, 0x5c, 0xc6, 0x47, 0x5b, - 0xab, 0xc3, 0x17, 0x1a, 0x80, 0xa3, 0x63, 0x17, - 0x49, 0x54, 0x95, 0x5d, 0x71, 0x01, 0xda, 0xb1, - 0x6a, 0xe8, 0x17, 0x91, 0x67, 0xe2, 0x14, 0x44, - 0xb4, 0x43, 0xa9, 0xea, 0xaa, 0x7c, 0x91, 0xde, - 0x36, 0xd1, 0x18, 0xc3, 0x9d, 0x38, 0x9f, 0x8d, - 0xd4, 0x46, 0x9a, 0x84, 0x6c, 0x9a, 0x26, 0x2b, - 0xf7, 0xfa, 0x18, 0x48, 0x7a, 0x79, 0xe8, 0xde, - 0x11, 0x69, 0x9e, 0x0b, 0x8f, 0xdf, 0x55, 0x7c, - 0xb4, 0x87, 0x19, 0xd4, 0x53, 0xba, 0x71, 0x30, - 0x56, 0x10, 0x9b, 0x93, 0xa2, 0x18, 0xc8, 0x96, - 0x75, 0xac, 0x19, 0x5f, 0xb4, 0xfb, 0x06, 0x63, - 0x9b, 0x37, 0x97, 0x14, 0x49, 0x55, 0xb3, 0xc9, - 0x32, 0x7d, 0x1a, 0xec, 0x00, 0x3d, 0x42, 0xec, - 0xd0, 0xea, 0x98, 0xab, 0xf1, 0x9f, 0xfb, 0x4a, - 0xf3, 0x56, 0x1a, 0x67, 0xe7, 0x7c, 0x35, 0xbf, - 0x15, 0xc5, 0x9c, 0x24, 0x12, 0xda, 0x88, 0x1d, - 0xb0, 0x2b, 0x1b, 0xfb, 0xce, 0xbf, 0xac, 0x51, - 0x52, 0xbc, 0x99, 0xbc, 0x3f, 0x1d, 0x15, 0xf7, - 0x71, 0x00, 0x1b, 0x70, 0x29, 0xfe, 0xdb, 0x02, - 0x8f, 0x8b, 0x85, 0x2b, 0xc4, 0x40, 0x7e, 0xb8, - 0x3f, 0x89, 0x1c, 0x9c, 0xa7, 0x33, 0x25, 0x4f, - 0xdd, 0x1e, 0x9e, 0xdb, 0x56, 0x91, 0x9c, 0xe9, - 0xfe, 0xa2, 0x1c, 0x17, 0x40, 0x72, 0x52, 0x1c, - 0x18, 0x31, 0x9a, 0x54, 0xb5, 0xd4, 0xef, 0xbe, - 0xbd, 0xdf, 0x1d, 0x8b, 0x69, 0xb1, 0xcb, 0xf2, - 0x5f, 0x48, 0x9f, 0xcc, 0x98, 0x13, 0x72, 0x54, - 0x7c, 0xf4, 0x1d, 0x00, 0x8e, 0xf0, 0xbc, 0xa1, - 0x92, 0x6f, 0x93, 0x4b, 0x73, 0x5e, 0x09, 0x0b, - 0x3b, 0x25, 0x1e, 0xb3, 0x3a, 0x36, 0xf8, 0x2e, - 0xd9, 0xb2, 0x9c, 0xf4, 0xcb, 0x94, 0x41, 0x88, - 0xfa, 0x0e, 0x1e, 0x38, 0xdd, 0x77, 0x8f, 0x7d, - 0x1c, 0x9d, 0x98, 0x7b, 0x28, 0xd1, 0x32, 0xdf, - 0xb9, 0x73, 0x1f, 0xa4, 0xf4, 0xb4, 0x16, 0x93, - 0x5b, 0xe4, 0x9d, 0xe3, 0x05, 0x16, 0xaf, 0x35, - 0x78, 0x58, 0x1f, 0x2f, 0x13, 0xf5, 0x61, 0xc0, - 0x66, 0x33, 0x61, 0x94, 0x1e, 0xab, 0x24, 0x9a, - 0x4b, 0xc1, 0x23, 0xf8, 0xd1, 0x5c, 0xd7, 0x11, - 0xa9, 0x56, 0xa1, 0xbf, 0x20, 0xfe, 0x6e, 0xb7, - 0x8a, 0xea, 0x23, 0x73, 0x36, 0x1d, 0xa0, 0x42, - 0x6c, 0x79, 0xa5, 0x30, 0xc3, 0xbb, 0x1d, 0xe0, - 0xc9, 0x97, 0x22, 0xef, 0x1f, 0xde, 0x39, 0xac, - 0x2b, 0x00, 0xa0, 0xa8, 0xee, 0x7c, 0x80, 0x0a, - 0x08, 0xbc, 0x22, 0x64, 0xf8, 0x9f, 0x4e, 0xff, - 0xe6, 0x27, 0xac, 0x2f, 0x05, 0x31, 0xfb, 0x55, - 0x4f, 0x6d, 0x21, 0xd7, 0x4c, 0x59, 0x0a, 0x70, - 0xad, 0xfa, 0xa3, 0x90, 0xbd, 0xfb, 0xb3, 0xd6, - 0x8e, 0x46, 0x21, 0x5c, 0xab, 0x18, 0x7d, 0x23, - 0x68, 0xd5, 0xa7, 0x1f, 0x5e, 0xbe, 0xc0, 0x81, - 0xcd, 0x3b, 0x20, 0xc0, 0x82, 0xdb, 0xe4, 0xcd, - 0x2f, 0xac, 0xa2, 0x87, 0x73, 0x79, 0x5d, 0x6b, - 0x0c, 0x10, 0x20, 0x4b, 0x65, 0x9a, 0x93, 0x9e, - 0xf2, 0x9b, 0xbe, 0x10, 0x88, 0x24, 0x36, 0x24, - 0x42, 0x99, 0x27, 0xa7, 0xeb, 0x57, 0x6d, 0xd3, - 0xa0, 0x0e, 0xa5, 0xe0, 0x1a, 0xf5, 0xd4, 0x75, - 0x83, 0xb2, 0x27, 0x2c, 0x0c, 0x16, 0x1a, 0x80, - 0x65, 0x21, 0xa1, 0x6f, 0xf9, 0xb0, 0xa7, 0x22, - 0xc0, 0xcf, 0x26, 0xb0, 0x25, 0xd5, 0x83, 0x6e, - 0x22, 0x58, 0xa4, 0xf7, 0xd4, 0x77, 0x3a, 0xc8, - 0x01, 0xe4, 0x26, 0x3b, 0xc2, 0x94, 0xf4, 0x3d, - 0xef, 0x7f, 0xa8, 0x70, 0x3f, 0x3a, 0x41, 0x97, - 0x46, 0x35, 0x25, 0x88, 0x76, 0x52, 0xb0, 0xb2, - 0xa4, 0xa2, 0xa7, 0xcf, 0x87, 0xf0, 0x09, 0x14, - 0x87, 0x1e, 0x25, 0x03, 0x91, 0x13, 0xc7, 0xe1, - 0x61, 0x8d, 0xa3, 0x40, 0x64, 0xb5, 0x7a, 0x43, - 0xc4, 0x63, 0x24, 0x9f, 0xb8, 0xd0, 0x5e, 0x0f, - 0x26, 0xf4, 0xa6, 0xd8, 0x49, 0x72, 0xe7, 0xa9, - 0x05, 0x48, 0x24, 0x14, 0x5f, 0x91, 0x29, 0x5c, - 0xdb, 0xe3, 0x9a, 0x6f, 0x92, 0x0f, 0xac, 0xc6, - 0x59, 0x71, 0x2b, 0x46, 0xa5, 0x4b, 0xa2, 0x95, - 0xbb, 0xe6, 0xa9, 0x01, 0x54, 0xe9, 0x1b, 0x33, - 0x98, 0x5a, 0x2b, 0xcd, 0x42, 0x0a, 0xd5, 0xc6, - 0x7e, 0xc9, 0xad, 0x8e, 0xb7, 0xac, 0x68, 0x64, - 0xdb, 0x27, 0x2a, 0x51, 0x6b, 0xc9, 0x4c, 0x28, - 0x39, 0xb0, 0xa8, 0x16, 0x9a, 0x6b, 0xf5, 0x8e, - 0x1a, 0x0c, 0x2a, 0xda, 0x8c, 0x88, 0x3b, 0x7b, - 0xf4, 0x97, 0xa4, 0x91, 0x71, 0x26, 0x8e, 0xd1, - 0x5d, 0xdd, 0x29, 0x69, 0x38, 0x4e, 0x7f, 0xf4, - 0xbf, 0x4a, 0xab, 0x2e, 0xc9, 0xec, 0xc6, 0x52, - 0x9c, 0xf6, 0x29, 0xe2, 0xdf, 0x0f, 0x08, 0xa7, - 0x7a, 0x65, 0xaf, 0xa1, 0x2a, 0xa9, 0xb5, 0x05, - 0xdf, 0x8b, 0x28, 0x7e, 0xf6, 0xcc, 0x91, 0x49, - 0x3d, 0x1c, 0xaa, 0x39, 0x07, 0x6e, 0x28, 0xef, - 0x1e, 0xa0, 0x28, 0xf5, 0x11, 0x8d, 0xe6, 0x1a, - 0xe0, 0x2b, 0xb6, 0xae, 0xfc, 0x33, 0x43, 0xa0, - 0x50, 0x29, 0x2f, 0x19, 0x9f, 0x40, 0x18, 0x57, - 0xb2, 0xbe, 0xad, 0x5e, 0x6e, 0xe2, 0xa1, 0xf1, - 0x91, 0x02, 0x2f, 0x92, 0x78, 0x01, 0x6f, 0x04, - 0x77, 0x91, 0xa9, 0xd1, 0x8d, 0xa7, 0xd2, 0xa6, - 0xd2, 0x7f, 0x2e, 0x0e, 0x51, 0xc2, 0xf6, 0xea, - 0x30, 0xe8, 0xac, 0x49, 0xa0, 0x60, 0x4f, 0x4c, - 0x13, 0x54, 0x2e, 0x85, 0xb6, 0x83, 0x81, 0xb9, - 0xfd, 0xcf, 0xa0, 0xce, 0x4b, 0x2d, 0x34, 0x13, - 0x54, 0x85, 0x2d, 0x36, 0x02, 0x45, 0xc5, 0x36, - 0xb6, 0x12, 0xaf, 0x71, 0xf3, 0xe7, 0x7c, 0x90, - 0x95, 0xae, 0x2d, 0xbd, 0xe5, 0x04, 0xb2, 0x65, - 0x73, 0x3d, 0xab, 0xfe, 0x10, 0xa2, 0x0f, 0xc7, - 0xd6, 0xd3, 0x2c, 0x21, 0xcc, 0xc7, 0x2b, 0x8b, - 0x34, 0x44, 0xae, 0x66, 0x3d, 0x65, 0x92, 0x2d, - 0x17, 0xf8, 0x2c, 0xaa, 0x2b, 0x86, 0x5c, 0xd8, - 0x89, 0x13, 0xd2, 0x91, 0xa6, 0x58, 0x99, 0x02, - 0x6e, 0xa1, 0x32, 0x84, 0x39, 0x72, 0x3c, 0x19, - 0x8c, 0x36, 0xb0, 0xc3, 0xc8, 0xd0, 0x85, 0xbf, - 0xaf, 0x8a, 0x32, 0x0f, 0xde, 0x33, 0x4b, 0x4a, - 0x49, 0x19, 0xb4, 0x4c, 0x2b, 0x95, 0xf6, 0xe8, - 0xec, 0xf7, 0x33, 0x93, 0xf7, 0xf0, 0xd2, 0xa4, - 0x0e, 0x60, 0xb1, 0xd4, 0x06, 0x52, 0x6b, 0x02, - 0x2d, 0xdc, 0x33, 0x18, 0x10, 0xb1, 0xa5, 0xf7, - 0xc3, 0x47, 0xbd, 0x53, 0xed, 0x1f, 0x10, 0x5d, - 0x6a, 0x0d, 0x30, 0xab, 0xa4, 0x77, 0xe1, 0x78, - 0x88, 0x9a, 0xb2, 0xec, 0x55, 0xd5, 0x58, 0xde, - 0xab, 0x26, 0x30, 0x20, 0x43, 0x36, 0x96, 0x2b, - 0x4d, 0xb5, 0xb6, 0x63, 0xb6, 0x90, 0x2b, 0x89, - 0xe8, 0x5b, 0x31, 0xbc, 0x6a, 0xf5, 0x0f, 0xc5, - 0x0a, 0xcc, 0xb3, 0xfb, 0x9b, 0x57, 0xb6, 0x63, - 0x29, 0x70, 0x31, 0x37, 0x8d, 0xb4, 0x78, 0x96, - 0xd7, 0xfb, 0xaf, 0x6c, 0x60, 0x0a, 0xdd, 0x2c, - 0x67, 0xf9, 0x36, 0xdb, 0x03, 0x79, 0x86, 0xdb, - 0x85, 0x6e, 0xb4, 0x9c, 0xf2, 0xdb, 0x3f, 0x7d, - 0xa6, 0xd2, 0x36, 0x50, 0xe4, 0x38, 0xf1, 0x88, - 0x40, 0x41, 0xb0, 0x13, 0x11, 0x9e, 0x4c, 0x2a, - 0xe5, 0xaf, 0x37, 0xcc, 0xcd, 0xfb, 0x68, 0x66, - 0x07, 0x38, 0xb5, 0x8b, 0x3c, 0x59, 0xd1, 0xc0, - 0x24, 0x84, 0x37, 0x47, 0x2a, 0xba, 0x1f, 0x35, - 0xca, 0x1f, 0xb9, 0x0c, 0xd7, 0x14, 0xaa, 0x9f, - 0x63, 0x55, 0x34, 0xf4, 0x9e, 0x7c, 0x5b, 0xba, - 0x81, 0xc2, 0xb6, 0xb3, 0x6f, 0xde, 0xe2, 0x1c, - 0xa2, 0x7e, 0x34, 0x7f, 0x79, 0x3d, 0x2c, 0xe9, - 0x44, 0xed, 0xb2, 0x3c, 0x8c, 0x9b, 0x91, 0x4b, - 0xe1, 0x03, 0x35, 0xe3, 0x50, 0xfe, 0xb5, 0x07, - 0x03, 0x94, 0xb7, 0xa4, 0xa1, 0x5c, 0x0c, 0xa1, - 0x20, 0x28, 0x35, 0x68, 0xb7, 0xbf, 0xc2, 0x54, - 0xfe, 0x83, 0x8b, 0x13, 0x7a, 0x21, 0x47, 0xce, - 0x7c, 0x11, 0x3a, 0x3a, 0x4d, 0x65, 0x49, 0x9d, - 0x9e, 0x86, 0xb8, 0x7d, 0xbc, 0xc7, 0xf0, 0x3b, - 0xbd, 0x3a, 0x3a, 0xb1, 0xaa, 0x24, 0x3e, 0xce, - 0x5b, 0xa9, 0xbc, 0xf2, 0x5f, 0x82, 0x83, 0x6c, - 0xfe, 0x47, 0x3b, 0x2d, 0x83, 0xe7, 0xa7, 0x20, - 0x1c, 0xd0, 0xb9, 0x6a, 0x72, 0x45, 0x1e, 0x86, - 0x3f, 0x6c, 0x3b, 0xa6, 0x64, 0xa6, 0xd0, 0x73, - 0xd1, 0xf7, 0xb5, 0xed, 0x99, 0x08, 0x65, 0xd9, - 0x78, 0xbd, 0x38, 0x15, 0xd0, 0x60, 0x94, 0xfc, - 0x9a, 0x2a, 0xba, 0x52, 0x21, 0xc2, 0x2d, 0x5a, - 0xb9, 0x96, 0x38, 0x9e, 0x37, 0x21, 0xe3, 0xaf, - 0x5f, 0x05, 0xbe, 0xdd, 0xc2, 0x87, 0x5e, 0x0d, - 0xfa, 0xeb, 0x39, 0x02, 0x1e, 0xe2, 0x7a, 0x41, - 0x18, 0x7c, 0xbb, 0x45, 0xef, 0x40, 0xc3, 0xe7, - 0x3b, 0xc0, 0x39, 0x89, 0xf9, 0xa3, 0x0d, 0x12, - 0xc5, 0x4b, 0xa7, 0xd2, 0x14, 0x1d, 0xa8, 0xa8, - 0x75, 0x49, 0x3e, 0x65, 0x77, 0x6e, 0xf3, 0x5f, - 0x97, 0xde, 0xbc, 0x22, 0x86, 0xcc, 0x4a, 0xf9, - 0xb4, 0x62, 0x3e, 0xee, 0x90, 0x2f, 0x84, 0x0c, - 0x52, 0xf1, 0xb8, 0xad, 0x65, 0x89, 0x39, 0xae, - 0xf7, 0x1f, 0x3f, 0x72, 0xb9, 0xec, 0x1d, 0xe2, - 0x15, 0x88, 0xbd, 0x35, 0x48, 0x4e, 0xa4, 0x44, - 0x36, 0x34, 0x3f, 0xf9, 0x5e, 0xad, 0x6a, 0xb1, - 0xd8, 0xaf, 0xb1, 0xb2, 0xa3, 0x03, 0xdf, 0x1b, - 0x71, 0xe5, 0x3c, 0x4a, 0xea, 0x6b, 0x2e, 0x3e, - 0x93, 0x72, 0xbe, 0x0d, 0x1b, 0xc9, 0x97, 0x98, - 0xb0, 0xce, 0x3c, 0xc1, 0x0d, 0x2a, 0x59, 0x6d, - 0x56, 0x5d, 0xba, 0x82, 0xf8, 0x8c, 0xe4, 0xcf, - 0xf3, 0xb3, 0x3d, 0x5d, 0x24, 0xe9, 0xc0, 0x83, - 0x11, 0x24, 0xbf, 0x1a, 0xd5, 0x4b, 0x79, 0x25, - 0x32, 0x98, 0x3d, 0xd6, 0xc3, 0xa8, 0xb7, 0xd0 -}; - -/* - * AES-CMAC-256 vectors - */ - -/* - * Subkey Generation - * K 603DEB10 15CA71BE 2B73AEF0 857D7781 - * 1F352C07 3B6108D7 2D9810A3 0914DFF4 - * AES-256(key,0) E568F681 94CF76D6 174D4CC0 4310A854 - * K1 CAD1ED03 299EEDAC 2E9A9980 8621502F - * K2 95A3DA06 533DDB58 5D353301 0C42A0D9 - */ -static const uint8_t cmac_256_key[32] = { - 0x60, 0x3D, 0xEB, 0x10, 0x15, 0xCA, 0x71, 0xBE, - 0x2B, 0x73, 0xAE, 0xF0, 0x85, 0x7D, 0x77, 0x81, - 0x1F, 0x35, 0x2C, 0x07, 0x3B, 0x61, 0x08, 0xD7, - 0x2D, 0x98, 0x10, 0xA3, 0x09, 0x14, 0xDF, 0xF4 -}; -static const uint8_t cmac_256_sub_key1[16] = { - 0xCA, 0xD1, 0xED, 0x03, 0x29, 0x9E, 0xED, 0xAC, - 0x2E, 0x9A, 0x99, 0x80, 0x86, 0x21, 0x50, 0x2F -}; -static const uint8_t cmac_256_sub_key2[16] = { - 0x95, 0xA3, 0xDA, 0x06, 0x53, 0x3D, 0xDB, 0x58, - 0x5D, 0x35, 0x33, 0x01, 0x0C, 0x42, 0xA0, 0xD9 -}; - -/* - * Example 1: len = 0 - * M - * AES-CMAC 028962F6 1B7BF89E FC6B551F 4667D983 - */ -static const uint8_t CMAC_256_T_1[16] = { - 0x02, 0x89, 0x62, 0xF6, 0x1B, 0x7B, 0xF8, 0x9E, - 0xFC, 0x6B, 0x55, 0x1F, 0x46, 0x67, 0xD9, 0x83 -}; - -/* - * Example 2: len = 16 - * M 6BC1BEE2 2E409F96 E93D7E11 7393172A - * AES-CMAC 28A7023F 452E8F82 BD4BF28D 8C37C35C - */ -static const uint8_t CMAC_256_T_2[16] = { - 0x28, 0xA7, 0x02, 0x3F, 0x45, 0x2E, 0x8F, 0x82, - 0xBD, 0x4B, 0xF2, 0x8D, 0x8C, 0x37, 0xC3, 0x5C -}; - -/* - * Example 3: len = 20 - * M 6BC1BEE2 2E409F96 E93D7E11 7393172A - * AE2D8A57 - * AES-CMAC 156727DC 0878944A 023C1FE0 3BAD6D93 - */ -static const uint8_t CMAC_256_T_3[16] = { - 0x15, 0x67, 0x27, 0xDC, 0x08, 0x78, 0x94, 0x4A, - 0x02, 0x3C, 0x1F, 0xE0, 0x3B, 0xAD, 0x6D, 0x93 -}; - -/* - * Example 4: len = 64 - * M 6BC1BEE2 2E409F96 E93D7E11 7393172A - * AE2D8A57 1E03AC9C 9EB76FAC 45AF8E51 - * 30C81C46 A35CE411 E5FBC119 1A0A52EF - * F69F2445 DF4F9B17 AD2B417B E66C3710 - * AES-CMAC E1992190 549F6ED5 696A2C05 6C315410 - */ -static const uint8_t CMAC_256_T_4[16] = { - 0xE1, 0x99, 0x21, 0x90, 0x54, 0x9F, 0x6E, 0xD5, - 0x69, 0x6A, 0x2C, 0x05, 0x6C, 0x31, 0x54, 0x10 -}; - - -static const struct cmac_rfc4493_vector { - const uint8_t *key; - const uint8_t *sub_key1; - const uint8_t *sub_key2; - const uint8_t *M; - size_t len; - const uint8_t *T; - size_t T_len; - enum cmac_type type; /* vector type - std or 3gpp */ -} cmac_vectors[] = { - { key, sub_key1, sub_key2, M, 0, T_1, 16, CMAC_128 }, - { key, sub_key1, sub_key2, M, 16, T_2, 16, CMAC_128 }, - { key, sub_key1, sub_key2, M, 40, T_3, 16, CMAC_128 }, - { key, sub_key1, sub_key2, M, 64, T_4, 16, CMAC_128 }, - { key, sub_key1, sub_key2, M, 0, T_1, 15, CMAC_128 }, - { key, sub_key1, sub_key2, M, 16, T_2, 15, CMAC_128 }, - { key, sub_key1, sub_key2, M, 40, T_3, 15, CMAC_128 }, - { key, sub_key1, sub_key2, M, 64, T_4, 15, CMAC_128 }, - { key, sub_key1, sub_key2, M, 0, T_1, 12, CMAC_128 }, - { key, sub_key1, sub_key2, M, 16, T_2, 12, CMAC_128 }, - { key, sub_key1, sub_key2, M, 40, T_3, 12, CMAC_128 }, - { key, sub_key1, sub_key2, M, 64, T_4, 12, CMAC_128 }, - { key, sub_key1, sub_key2, M, 0, T_1, 4, CMAC_128 }, - { key, sub_key1, sub_key2, M, 16, T_2, 4, CMAC_128 }, - { key, sub_key1, sub_key2, M, 40, T_3, 4, CMAC_128 }, - { key, sub_key1, sub_key2, M, 64, T_4, 4, CMAC_128 }, - { key, sub_key1, sub_key2, M, 8, T_5, 16, CMAC_128 }, -}; - -static const struct cmac_rfc4493_vector cmac_256_vectors[] = { - { cmac_256_key, cmac_256_sub_key1, cmac_256_sub_key2, M, 0, - CMAC_256_T_1, 16, CMAC_256 }, - { cmac_256_key, cmac_256_sub_key1, cmac_256_sub_key2, M, 16, - CMAC_256_T_2, 16, CMAC_256 }, - { cmac_256_key, cmac_256_sub_key1, cmac_256_sub_key2, M, 20, - CMAC_256_T_3, 16, CMAC_256 }, - { cmac_256_key, cmac_256_sub_key1, cmac_256_sub_key2, M, 64, - CMAC_256_T_4, 16, CMAC_256 }, -}; - -static const struct cmac_rfc4493_vector cmac_3gpp_vectors[] = { - { EIA2_128_K_1, EIA2_128_SK1_1, EIA2_128_SK2_1, - EIA2_128_M_1, 122, EIA2_128_T_1, 4, CMAC_128_BITLEN }, - { EIA2_128_K_2, EIA2_128_SK1_2, EIA2_128_SK2_2, - EIA2_128_M_2, 128, EIA2_128_T_2, 4, CMAC_128_BITLEN }, - { EIA2_128_K_3, EIA2_128_SK1_3, EIA2_128_SK2_3, - EIA2_128_M_3, 318, EIA2_128_T_3, 4, CMAC_128_BITLEN }, - { EIA2_128_K_4, EIA2_128_SK1_4, EIA2_128_SK2_4, - EIA2_128_M_4, 575, EIA2_128_T_4, 4, CMAC_128_BITLEN }, - { EIA2_128_K_5, EIA2_128_SK1_5, EIA2_128_SK2_5, - EIA2_128_M_5, 832, EIA2_128_T_5, 4, CMAC_128_BITLEN }, - { EIA2_128_K_6, EIA2_128_SK1_6, EIA2_128_SK2_6, - EIA2_128_M_6, 447, EIA2_128_T_6, 4, CMAC_128_BITLEN }, - { EIA2_128_K_7, EIA2_128_SK1_7, EIA2_128_SK2_7, - EIA2_128_M_7, 2622, EIA2_128_T_7, 4, CMAC_128_BITLEN }, - { EIA2_128_K_8, EIA2_128_SK1_8, EIA2_128_SK2_8, - EIA2_128_M_8, 16512, EIA2_128_T_8, 4, CMAC_128_BITLEN }, -}; - -static int -cmac_job_ok(const struct cmac_rfc4493_vector *vec, - const struct IMB_JOB *job, - const uint8_t *auth, - const uint8_t *padding, - const size_t sizeof_padding) -{ - const size_t auth_len = job->auth_tag_output_len_in_bytes; - - if (job->status != IMB_STATUS_COMPLETED) { - printf("%d Error status:%d", __LINE__, job->status); - return 0; - } - - /* hash checks */ - if (memcmp(padding, &auth[sizeof_padding + auth_len], - sizeof_padding)) { - printf("hash overwrite tail\n"); - hexdump(stderr, "Target", - &auth[sizeof_padding + auth_len], sizeof_padding); - return 0; - } - - if (memcmp(padding, &auth[0], sizeof_padding)) { - printf("hash overwrite head\n"); - hexdump(stderr, "Target", &auth[0], sizeof_padding); - return 0; - } - - if (memcmp(vec->T, &auth[sizeof_padding], auth_len)) { - printf("hash mismatched\n"); - hexdump(stderr, "Received", &auth[sizeof_padding], - auth_len); - hexdump(stderr, "Expected", vec->T, - auth_len); - return 0; - } - return 1; -} - -static int -test_cmac(struct IMB_MGR *mb_mgr, - const struct cmac_rfc4493_vector *vec, - const int dir, - const int num_jobs, - const enum cmac_type type) -{ - DECLARE_ALIGNED(uint32_t expkey[4*15], 16); - DECLARE_ALIGNED(uint32_t dust[4*15], 16); - uint32_t skey1[4], skey2[4]; - struct IMB_JOB *job; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - int i = 0, jobs_rx = 0, ret = -1; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - auths[i] = malloc(16 + (sizeof(padding) * 2)); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - - memset(auths[i], -1, 16 + (sizeof(padding) * 2)); - } - - if ((type == CMAC_128) || (type == CMAC_128_BITLEN)) { - IMB_AES_KEYEXP_128(mb_mgr, vec->key, expkey, dust); - IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, expkey, skey1, skey2); - } else { /* AES-CMAC-256 */ - IMB_AES_KEYEXP_256(mb_mgr, vec->key, expkey, dust); - IMB_AES_CMAC_SUBKEY_GEN_256(mb_mgr, expkey, skey1, skey2); - } - - if (memcmp(vec->sub_key1, skey1, sizeof(skey1))) { - printf("sub-key1 mismatched\n"); - hexdump(stderr, "Received", &skey1[0], sizeof(skey1)); - hexdump(stderr, "Expected", vec->sub_key1, sizeof(skey1)); - goto end; - } - - if (memcmp(vec->sub_key2, skey2, sizeof(skey2))) { - printf("sub-key2 mismatched\n"); - hexdump(stderr, "Received", &skey2[0], sizeof(skey2)); - hexdump(stderr, "Expected", vec->sub_key2, sizeof(skey2)); - goto end; - } - - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - /** - * Submit all jobs then flush any outstanding jobs - */ - for (i = 0; i < num_jobs; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - job->cipher_direction = dir; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->cipher_mode = IMB_CIPHER_NULL; - - switch (type) { - case CMAC_128: - job->hash_alg = IMB_AUTH_AES_CMAC; - job->msg_len_to_hash_in_bytes = vec->len; - break; - case CMAC_128_BITLEN: - job->hash_alg = IMB_AUTH_AES_CMAC_BITLEN; - /* check for std or 3gpp vectors - scale len if necessary */ - if (vec->type == CMAC_128) - job->msg_len_to_hash_in_bits = vec->len * 8; - else - job->msg_len_to_hash_in_bits = vec->len; - break; - case CMAC_256: - job->hash_alg = IMB_AUTH_AES_CMAC_256; - job->msg_len_to_hash_in_bytes = vec->len; - break; - default: - printf("Invalid CMAC type specified\n"); - goto end; - } - job->u.CMAC._key_expanded = expkey; - job->u.CMAC._skey1 = skey1; - job->u.CMAC._skey2 = skey2; - job->src = vec->M; - job->hash_start_src_offset_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->T_len; - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job) { - jobs_rx++; - if (num_jobs < 4) { - printf("%d Unexpected return from submit_job\n", - __LINE__); - goto end; - } - if (!cmac_job_ok(vec, job, job->user_data, padding, - sizeof(padding))) - goto end; - } - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - jobs_rx++; - - if (!cmac_job_ok(vec, job, job->user_data, padding, - sizeof(padding))) - goto end; - } - - if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); - goto end; - } - - /** - * Submit each job and flush immediately - */ - for (i = 0; i < num_jobs; i++) { - struct IMB_JOB *first_job = NULL; - - job = IMB_GET_NEXT_JOB(mb_mgr); - first_job = job; - - job->cipher_direction = dir; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->cipher_mode = IMB_CIPHER_NULL; - - switch (type) { - case CMAC_128: - job->hash_alg = IMB_AUTH_AES_CMAC; - job->msg_len_to_hash_in_bytes = vec->len; - break; - case CMAC_128_BITLEN: - job->hash_alg = IMB_AUTH_AES_CMAC_BITLEN; - /* check for std or 3gpp vectors - scale len if necessary */ - if (vec->type == CMAC_128) - job->msg_len_to_hash_in_bits = vec->len * 8; - else - job->msg_len_to_hash_in_bits = vec->len; - break; - case CMAC_256: - job->hash_alg = IMB_AUTH_AES_CMAC_256; - job->msg_len_to_hash_in_bytes = vec->len; - break; - default: - printf("Invalid CMAC type specified\n"); - goto end; - } - job->u.CMAC._key_expanded = expkey; - job->u.CMAC._skey1 = skey1; - job->u.CMAC._skey2 = skey2; - job->src = vec->M; - job->hash_start_src_offset_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->T_len; - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job != NULL) { - printf("Received job, expected NULL\n"); - goto end; - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - if (job != first_job) { - printf("Invalid return job received\n"); - goto end; - } - if (!cmac_job_ok(vec, job, job->user_data, padding, - sizeof(padding))) - goto end; - } - } - - ret = 0; - - end: - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static void -test_cmac_std_vectors(struct IMB_MGR *mb_mgr, - struct test_suite_context *ctx, - const int num_jobs) -{ - const int vectors_cnt = DIM(cmac_vectors); - int vect; - - printf("AES-CMAC-128 standard test vectors (N jobs = %d):\n", num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("Standard CMAC-128 vector [%d/%d] M len: %d, T len:%d\n", - vect, vectors_cnt, - (int) cmac_vectors[idx].len, - (int) cmac_vectors[idx].T_len); -#else - printf("."); -#endif - - if (test_cmac(mb_mgr, &cmac_vectors[idx], - IMB_DIR_ENCRYPT, num_jobs, CMAC_128)) { - printf("error #%d encrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - if (test_cmac(mb_mgr, &cmac_vectors[idx], - IMB_DIR_DECRYPT, num_jobs, CMAC_128)) { - printf("error #%d decrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - } - printf("\n"); -} - -static void -test_cmac_256_std_vectors(struct IMB_MGR *mb_mgr, - struct test_suite_context *ctx, - const int num_jobs) -{ - const int vectors_cnt = DIM(cmac_256_vectors); - int vect; - - printf("AES-CMAC-256 standard test vectors (N jobs = %d):\n", num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("Standard CMAC-256 vector [%d/%d] M len: %d, T len:%d\n", - vect, vectors_cnt, - (int) cmac_256_vectors[idx].len, - (int) cmac_256_vectors[idx].T_len); -#else - printf("."); -#endif - - if (test_cmac(mb_mgr, &cmac_256_vectors[idx], - IMB_DIR_ENCRYPT, num_jobs, CMAC_256)) { - printf("error #%d encrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - if (test_cmac(mb_mgr, &cmac_256_vectors[idx], - IMB_DIR_DECRYPT, num_jobs, CMAC_256)) { - printf("error #%d decrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - } - printf("\n"); -} - -static void -test_cmac_bitlen_std_vectors(struct IMB_MGR *mb_mgr, - struct test_suite_context *ctx, - const int num_jobs) -{ - const int vectors_cnt = sizeof(cmac_vectors) / sizeof(cmac_vectors[0]); - int vect; - - printf("AES-CMAC-128 BITLEN standard test vectors " - "(N jobs = %d):\n", num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("Standard vector [%d/%d] M len: %d (bits), " - "T len:%d\n", - vect, vectors_cnt, - (int) cmac_vectors[idx].len * 8, - (int) cmac_vectors[idx].T_len); -#else - printf("."); -#endif - - if (test_cmac(mb_mgr, &cmac_vectors[idx], - IMB_DIR_ENCRYPT, num_jobs, CMAC_128_BITLEN)) { - printf("error #%d encrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - if (test_cmac(mb_mgr, &cmac_vectors[idx], - IMB_DIR_DECRYPT, num_jobs, CMAC_128_BITLEN)) { - printf("error #%d decrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - } - printf("\n"); -} - -static void -test_cmac_bitlen_3gpp_vectors(struct IMB_MGR *mb_mgr, - struct test_suite_context *ctx, - const int num_jobs) -{ - const int vectors_cnt = - sizeof(cmac_3gpp_vectors) / sizeof(cmac_3gpp_vectors[0]); - int vect; - - printf("AES-CMAC-128 BITLEN 3GPP test vectors (N jobs = %d):\n", - num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("3GPP vector [%d/%d] M len: %d (bits), " - "T len:%d (bytes)\n", - vect, vectors_cnt, - (int) cmac_3gpp_vectors[idx].len, - (int) cmac_3gpp_vectors[idx].T_len); -#else - printf("."); -#endif - - if (test_cmac(mb_mgr, &cmac_3gpp_vectors[idx], - IMB_DIR_ENCRYPT, num_jobs, CMAC_128_BITLEN)) { - printf("error #%d encrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - if (test_cmac(mb_mgr, &cmac_3gpp_vectors[idx], - IMB_DIR_DECRYPT, num_jobs, CMAC_128_BITLEN)) { - printf("error #%d decrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - } - printf("\n"); -} - -int -cmac_test(struct IMB_MGR *mb_mgr) -{ - int i, errors = 0; - struct test_suite_context ctx; - - /* CMAC 128 with standard vectors */ - test_suite_start(&ctx, "AES-CMAC-128"); - for (i = 1; i < 20; i++) - test_cmac_std_vectors(mb_mgr, &ctx, i); - errors += test_suite_end(&ctx); - - /* CMAC 128 BITLEN with standard vectors */ - test_suite_start(&ctx, "AES-CMAC-128-BIT-LENGTH"); - for (i = 1; i < 20; i++) - test_cmac_bitlen_std_vectors(mb_mgr, &ctx, i); - - /* CMAC 128 BITLEN with 3GPP vectors */ - for (i = 1; i < 20; i++) - test_cmac_bitlen_3gpp_vectors(mb_mgr, &ctx, i); - errors += test_suite_end(&ctx); - - /* CMAC 256 with standard vectors */ - test_suite_start(&ctx, "AES-CMAC-256"); - for (i = 1; i < 20; i++) - test_cmac_256_std_vectors(mb_mgr, &ctx, i); - errors += test_suite_end(&ctx); - - return errors; -} diff --git a/test/cmake/unix.cmake b/test/cmake/unix.cmake new file mode 100644 index 0000000000000000000000000000000000000000..a035677fdb6a189d1e880e95c8dd019efff691a7 --- /dev/null +++ b/test/cmake/unix.cmake @@ -0,0 +1,64 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# Test application CMake Unix config +# ############################################################################## + +set(IPSEC_MB_LIB IPSec_MB) + +# set NASM flags +set(CMAKE_ASM_NASM_FLAGS "-felf64 -Xgnu -gdwarf -DLINUX -D__linux__") + +# set compiler definitions +set(APP_DEFINES LINUX _GNU_SOURCE) + +# set C compiler flags +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(CMAKE_C_FLAGS + "-W -Wall -Wextra -Wmissing-declarations \ +-Wpointer-arith -Wcast-qual -Wundef -Wwrite-strings -Wformat \ +-Wformat-security -Wunreachable-code -Wmissing-noreturn -Wsign-compare \ +-Wno-endif-labels -Wstrict-prototypes -Wmissing-prototypes \ +-Wold-style-definition -fno-delete-null-pointer-checks -fwrapv -std=c99 \ +-Wno-unused-parameter -Wno-unused-variable -Wno-unused-function") +else() + set(CMAKE_C_FLAGS + "-W -Wall -Wextra -Wmissing-declarations \ +-Wpointer-arith -Wcast-qual -Wundef -Wwrite-strings -Wformat \ +-Wformat-security -Wunreachable-code -Wmissing-noreturn -Wsign-compare \ +-Wno-endif-labels -Wstrict-prototypes -Wmissing-prototypes \ +-Wold-style-definition -fno-delete-null-pointer-checks -fwrapv -std=c99") +endif() +set(CMAKE_C_FLAGS_DEBUG "-O0 -DDEBUG -g") +set(CMAKE_C_FLAGS_RELEASE "-O3") +set(CMAKE_EXE_LINKER_FLAGS "-fPIE -z noexecstack -z relro -z now") +set(CMAKE_EXE_LINKER_FLAGS_DEBUG "-g") + +# -fno-strict-overflow is not supported by clang +if(CMAKE_COMPILER_IS_GNUCC) + string(APPEND CMAKE_C_FLAGS " -fno-strict-overflow") +endif() + diff --git a/test/cmake/windows.cmake b/test/cmake/windows.cmake new file mode 100644 index 0000000000000000000000000000000000000000..5982eee08af9ec93bbd437b47fbc1a4e428ae96f --- /dev/null +++ b/test/cmake/windows.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ############################################################################## +# Test application CMake Windows config +# ############################################################################## + +set(IPSEC_MB_LIB libIPSec_MB) + +# set NASM flags +set(CMAKE_ASM_NASM_FLAGS "-Werror -fwin64 -Xvc -DWIN_ABI") + +# set C compiler flags +set(CMAKE_C_FLAGS "/nologo /D_CRT_SECURE_NO_WARNINGS /Y- /W3 /WX- /Gm- /fp:precise /EHsc /std:c11 ${EXTRA_CFLAGS}") +set(CMAKE_C_FLAGS_DEBUG "/Od /DDEBUG /Z7") +set(CMAKE_C_FLAGS_RELEASE "/O2 /Oi") +set(CMAKE_EXE_LINKER_FLAGS_DEBUG "/debug") + diff --git a/test/wycheproof/Makefile b/test/common/common.mk similarity index 69% rename from test/wycheproof/Makefile rename to test/common/common.mk index ef9be7ab4194eec53dee2f66737874cad7feb6de..311b7cc130ba2550fa179354e6b3a3af9da10b99 100644 --- a/test/wycheproof/Makefile +++ b/test/common/common.mk @@ -1,5 +1,5 @@ # -# Copyright (c) 2022, Intel Corporation +# Copyright (c) 2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -25,19 +25,37 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -APP := wycheproof +ARCH = $(shell uname -m) + INSTPATH ?= /usr/include/ipsec-mb.h LIB_DIR ?= ../../lib +ifeq ($(ARCH),x86_64) +USE_YASM ?= n +YASM ?= yasm +NASM ?= nasm +endif # x86_64 + MINGW ?= $(shell $(CC) -dM -E - < /dev/null | grep -i mingw | wc -l | sed 's/^ *//') -CFLAGS = -MMD -D_GNU_SOURCE -DNO_COMPAT_IMB_API_053 \ +ifeq ($(ARCH),x86_64) +CFLAGS = -MMD -D_GNU_SOURCE \ -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ -Wcast-qual -Wundef -Wwrite-strings \ -Wformat -Wformat-security \ -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ - -fno-delete-null-pointer-checks -fwrapv + -fno-delete-null-pointer-checks -fwrapv -std=c99 +else +CFLAGS = -MMD -D_GNU_SOURCE \ + -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ + -Wcast-qual -Wundef -Wwrite-strings \ + -Wformat -Wformat-security \ + -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ + -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ + -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function \ + -fno-delete-null-pointer-checks -fwrapv -std=c99 +endif # -fno-strict-overflow is not supported by clang ifneq ($(CC),clang) @@ -46,6 +64,7 @@ endif # if "-z ibt" is supported then assume "-z shstk, -z cet-report=error" are also supported # "-fcf-protection" needs to be checked separately +ifeq ($(ARCH),x86_64) ifeq ($(MINGW),0) CC_HAS_CET = $(and $(shell $(CC) --target-help 2> /dev/null | grep -m1 -e "-z ibt" | wc -l), \ $(shell $(CC) --help=common 2> /dev/null | grep -m1 -e "-fcf-protection" | wc -l)) @@ -56,24 +75,40 @@ ifeq ($(CC_HAS_CET),1) CFLAGS += -fcf-protection=full endif +YASM_FLAGS := -f x64 -f elf64 -X gnu -g dwarf2 -DLINUX -D__linux__ +ifeq ($(MINGW),0) +CFLAGS += -DLINUX +NASM_FLAGS := -Werror -felf64 -Xgnu -gdwarf -DLINUX -D__linux__ +else +NASM_FLAGS := -Werror -fwin64 -Xvc -gcv8 -DWIN_ABI +endif +else # x86_64 +ifeq ($(MINGW),0) +CFLAGS += -DLINUX +endif +endif # x86_64 + ifeq ($(MINGW),0) LDFLAGS = -fPIE -z noexecstack -z relro -z now else LDFLAGS = -fPIE endif +ifeq ($(ARCH),x86_64) ifeq ($(CC_HAS_CET),1) LDFLAGS += -fcf-protection=full -Wl,-z,ibt -Wl,-z,shstk -Wl,-z,cet-report=error endif +endif # x86_64 LDLIBS = -lIPSec_MB ifeq ("$(shell test -r $(INSTPATH) && echo -n yes)","yes") # library installed -CFLAGS += +$(info INFO: Using system installed library version.) +CFLAGS += -I../include/ else # library not installed -CFLAGS += -I../../lib +CFLAGS += -I../../lib -I../include/ LDFLAGS += -L$(LIB_DIR) endif @@ -89,40 +124,10 @@ CFLAGS += -O2 endif endif -OBJ_FILES = aes_gcm_test.json.o aes_ccm_test.json.o \ - chacha20_poly1305_test.json.o \ - aes_cmac_test.json.o gmac_test.json.o gmac_test.json.o \ - hmac_sha1_test.json.o hmac_sha224_test.json.o hmac_sha256_test.json.o \ - hmac_sha384_test.json.o hmac_sha512_test.json.o \ - wycheproof.o - -all: $(APP) - -$(APP): $(OBJ_FILES) - -.PHONY: clean -clean: - -rm -f $(OBJ_FILES) *.d $(APP) - -# style check section -CHECKPATCH?=checkpatch.pl -CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,AVOID_EXTERNS,COMPLEX_MACRO,USE_FUNC,CONSTANT_COMPARISON,MISSING_SPACE,NEW_TYPEDEFS - -%.c_style_check : %.c - $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< - -%.h_style_check : %.h - $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< - -SOURCES_ALL := $(wildcard *.[ch]) -SOURCES_STYLE := $(foreach infile,$(SOURCES_ALL),$(infile)_style_check) +ifneq ($(PIN_CEC_ROOT),) +CFLAGS += -I$(PIN_CEC_ROOT)/include -DPIN_BASED_CEC +endif -.PHONY: style -style: $(SOURCES_STYLE) +# list of present dependency files +DEP_FILES = $(wildcard ./*.d) -# if target not clean or rinse then make dependencies -ifneq ($(MAKECMDGOALS),clean) -ifneq ($(MAKECMDGOALS),style) --include $(wildcard *.d) -endif -endif diff --git a/test/utils.c b/test/common/utils.c similarity index 94% rename from test/utils.c rename to test/common/utils.c index 0d931398c58a4619e637ea5d52e6aeecc12cff63..29a3519f5431ae53f19aba0ccfddb78eebab8aa4 100644 --- a/test/utils.c +++ b/test/common/utils.c @@ -35,6 +35,12 @@ #include "utils.h" #include +/** + * If not zero then modules need to limit amount of printed information + * (even in debug compilation mode). + */ +int quiet_mode = 0; + /** * @brief Simplistic memory copy (intentionally not using libc) * @@ -173,6 +179,7 @@ update_flags_and_archs(const char *arg, return -1; } +#ifndef __aarch64__ if (strcmp(arg, "--no-avx512") == 0) arch_support[IMB_ARCH_AVX512] = 0; else if (strcmp(arg, "--no-avx2") == 0) @@ -181,10 +188,6 @@ update_flags_and_archs(const char *arg, arch_support[IMB_ARCH_AVX] = 0; else if (strcmp(arg, "--no-sse") == 0) arch_support[IMB_ARCH_SSE] = 0; - else if (strcmp(arg, "--no-aarch64") ==0) - arch_support[IMB_ARCH_AARCH64] = 0; - else if (strcmp(arg, "--no-sve256") ==0) - arch_support[IMB_ARCH_SVE256] = 0; else if (strcmp(arg, "--aesni-emu") == 0) arch_support[IMB_ARCH_NOAESNI] = 1; else if (strcmp(arg, "--no-aesni-emu") == 0) @@ -197,6 +200,16 @@ update_flags_and_archs(const char *arg, *flags &= (~IMB_FLAG_GFNI_OFF); else if (strcmp(arg, "--gfni-off") == 0) *flags |= IMB_FLAG_GFNI_OFF; +#else + if (strcmp(arg, "--no-aarch64") ==0) + arch_support[IMB_ARCH_AARCH64] = 0; + else if (strcmp(arg, "--no-sve256") ==0) + arch_support[IMB_ARCH_SVE256] = 0; + else if (strcmp(arg, "--aesni-emu") == 0) + arch_support[IMB_ARCH_NOAESNI] = 1; + else if (strcmp(arg, "--no-aesni-emu") == 0) + arch_support[IMB_ARCH_NOAESNI] = 0; +#endif else match = 0; return match; @@ -214,21 +227,18 @@ update_flags_and_archs(const char *arg, int detect_arch(uint8_t arch_support[IMB_ARCH_NUM]) { +#ifndef __aarch64__ const uint64_t detect_sse = IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; const uint64_t detect_avx = IMB_FEATURE_AVX | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; const uint64_t detect_avx2 = IMB_FEATURE_AVX2 | detect_avx; const uint64_t detect_avx512 = IMB_FEATURE_AVX512_SKX | detect_avx2; - + const uint64_t detect_noaesni = IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV; +#else const uint64_t detect_aarch64 = IMB_FEATURE_AARCH64 | IMB_FEATURE_AESNI; const uint64_t detect_sve256 = IMB_FEATURE_AARCH64 | IMB_FEATURE_SVE256; -#ifdef __x86_64__ - const uint64_t detect_noaesni = IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV; -#endif - -#ifdef __aarch64__ const uint64_t detect_noaesni = IMB_FEATURE_AARCH64 | IMB_FEATURE_ASIMD; #endif @@ -249,6 +259,7 @@ detect_arch(uint8_t arch_support[IMB_ARCH_NUM]) return -1; } +#ifndef __aarch64__ if ((p_mgr->features & detect_avx512) != detect_avx512) arch_support[IMB_ARCH_AVX512] = 0; @@ -260,26 +271,29 @@ detect_arch(uint8_t arch_support[IMB_ARCH_NUM]) if ((p_mgr->features & detect_sse) != detect_sse) arch_support[IMB_ARCH_SSE] = 0; - - if ((p_mgr->features & detect_noaesni) != detect_noaesni) - arch_support[IMB_ARCH_NOAESNI] = 0; - +#else if ((p_mgr->features & detect_aarch64) != detect_aarch64) arch_support[IMB_ARCH_AARCH64] = 0; - if ((p_mgr->features & detect_sve256) != detect_sve256) { + if ((p_mgr->features & detect_sve256) != detect_sve256) arch_support[IMB_ARCH_SVE256] = 0; - } +#endif + + if ((p_mgr->features & detect_noaesni) != detect_noaesni) + arch_support[IMB_ARCH_NOAESNI] = 0; free_mb_mgr(p_mgr); if (arch_support[IMB_ARCH_NOAESNI] == 0 && +#ifndef __aarch64__ arch_support[IMB_ARCH_SSE] == 0 && arch_support[IMB_ARCH_AVX] == 0 && arch_support[IMB_ARCH_AVX2] == 0 && - arch_support[IMB_ARCH_AVX512] == 0 && + arch_support[IMB_ARCH_AVX512] == 0) { +#else arch_support[IMB_ARCH_AARCH64] == 0 && arch_support[IMB_ARCH_SVE256] == 0) { +#endif fprintf(stderr, "No available architecture detected!\n"); return -1; } @@ -297,16 +311,19 @@ void print_tested_arch(const uint64_t features, const IMB_ARCH arch) { static const char *arch_str_tab[IMB_ARCH_NUM] = { - "NONE", "NO-AESNI", "SSE", "AVX", "AVX2", "AVX512", "AARCH64", "SVE256" +#ifndef __aarch64__ + "NONE", "NO-AESNI", "SSE", "AVX", "AVX2", "AVX512" +#else + "NONE", "NO-AESNI", "AARCH64", "SVE256" +#endif }; const char *feat = ""; switch (arch) { +#ifndef __aarch64__ case IMB_ARCH_NOAESNI: case IMB_ARCH_AVX2: case IMB_ARCH_AVX: - case IMB_ARCH_AARCH64: - case IMB_ARCH_SVE256: break; case IMB_ARCH_SSE: if (features & IMB_FEATURE_SHANI) { @@ -320,6 +337,12 @@ print_tested_arch(const uint64_t features, const IMB_ARCH arch) (features & IMB_FEATURE_VPCLMULQDQ)) feat = "-VAES-GFNI-VCLMUL"; break; +#else + case IMB_ARCH_NOAESNI: + case IMB_ARCH_AARCH64: + case IMB_ARCH_SVE256: + break; +#endif default: printf("Invalid component\n"); return; diff --git a/test/common/win_x64_common.mk b/test/common/win_x64_common.mk new file mode 100644 index 0000000000000000000000000000000000000000..4b6c902c83b9fc2d76dfa0a5c0b46ecf742c4cdd --- /dev/null +++ b/test/common/win_x64_common.mk @@ -0,0 +1,75 @@ +# +# Copyright (c) 2017-2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +INSTNAME = ipsec-mb + +!if !defined(PREFIX) +PREFIX = C:\Program Files +!endif + +!if exist("$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib") +IPSECLIB = "$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib" +INCDIR = -I"$(PREFIX)\$(INSTNAME)" +!else +!if !defined(LIB_DIR) +LIB_DIR = ..\..\lib +!endif +IPSECLIB = "$(LIB_DIR)\libIPSec_MB.lib" +INCDIR = -I$(LIB_DIR) -I..\include +!endif + +!if !defined(DEBUG_OPT) +DEBUG_OPT = /Od +!endif + +!ifdef DEBUG +DCFLAGS = $(DEBUG_OPT) /DDEBUG /Z7 +DLFLAGS = /debug +!else +DCFLAGS = /O2 /Oi +DLFLAGS = +!endif + +# compiler +CC = cl + +# _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure snprintf() being used +CFLAGS = /nologo /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) /std:c11 + +#linker +LNK = link +# XVALID_LFLAGS = /out:$(XVALID_APP).exe $(DLFLAGS) + +AS = nasm +AFLAGS = -Werror -fwin64 -Xvc -DWIN_ABI + +# dependency +!ifndef DEPTOOL +DEPTOOL = ..\..\mkdep.bat +!endif +DEPFLAGS = $(INCDIR) + diff --git a/test/fuzz-app/Makefile b/test/fuzz-app/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fa906739c942c08596c9459dfba586858f23165d --- /dev/null +++ b/test/fuzz-app/Makefile @@ -0,0 +1,72 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +FUZZ_APP := imb-fuzz-api +DIRECT_FUZZ_APP := imb-fuzz-direct-api + +# include common options +include ../common/common.mk + +FUZZ_CFLAGS = -I../../lib -L../../lib $(CFLAGS) -g -fsanitize=fuzzer,address,leak + +# fuzz modules +FUZZ_SOURCES := job_api_fuzz_test.c +DIRECT_FUZZ_SOURCES := direct_api_fuzz_test.c + +# targets come here +all: $(FUZZ_APP) $(DIRECT_FUZZ_APP) + +$(FUZZ_APP): $(FUZZ_SOURCES) +ifeq ($(MINGW),0) +ifneq (, $(shell which clang)) + clang $(FUZZ_CFLAGS) $(FUZZ_SOURCES) $(LDLIBS) -o $@ +else + @echo "Clang is not installed. $(FUZZ_APP) is not built" +endif +else + @echo "Fuzzing not supported on MinGW build. $(FUZZ_APP) is not built" +endif + +$(DIRECT_FUZZ_APP): $(DIRECT_FUZZ_SOURCES) +ifeq ($(MINGW),0) +ifneq (, $(shell which clang)) + clang $(FUZZ_CFLAGS) $(DIRECT_FUZZ_SOURCES) $(LDLIBS) -o $@ +else + @echo "Clang is not installed. $(DIRECT_FUZZ_APP) is not built" +endif +else + @echo "Fuzzing not supported on MinGW build. $(DIRECT_FUZZ_APP) is not built" +endif + +.PHONY: clean +clean: + -rm -f $(DEP_FILES) $(FUZZ_APP) $(DIRECT_FUZZ_APP) + +# if target not clean then make dependencies +ifneq ($(MAKECMDGOALS),clean) +-include $(wildcard *.d) +endif diff --git a/test/fuzz-app/README.md b/test/fuzz-app/README.md new file mode 100644 index 0000000000000000000000000000000000000000..657fdb25ad81f5e8f61b2685161a42aa12445a51 --- /dev/null +++ b/test/fuzz-app/README.md @@ -0,0 +1,39 @@ +# Intel(R) Multi-Buffer Crypto for IPsec Library - Fuzz Test Applications + +## Contents + +- Overview +- Dependencies +- Usage + + +## Overview + +The fuzz test applications aim to discover defects in the library by passing randomly +generated data to the library API's. Currently there are two fuzzing applications, the +`imb-fuzz-api` application targets job and burst API and `imb-fuzz-direct-api` targets +the direct API. + + +## Dependencies +- clang +- libfuzzer + +## Usage + +Before running the application, ensure the library is installed by following the instructions +in the [README](https://github.com/intel/intel-ipsec-mb/tree/main/test#library-installation). +**Note:** The library must be compiled with SAFE_PARAM option enabled (default setting). +[CLANG/LLVM libFuzzer](https://llvm.org/docs/LibFuzzer.html) package is required for building and running the fuzz applications. +### Linux +To fuzz the library job and burst API: +`./imb-fuzz-api` + +To fuzz the library direct API: +`./imb-fuzz-direct-api` + +To display an extensive help page for libfuzzer options: +`./imb-fuzz-api -help=1` + +### Windows +Not currently supported. diff --git a/test/fuzz-app/direct_api_fuzz_test.c b/test/fuzz-app/direct_api_fuzz_test.c new file mode 100644 index 0000000000000000000000000000000000000000..91b4bd1dea5e546725bb5573526dddcf148c1c5a --- /dev/null +++ b/test/fuzz-app/direct_api_fuzz_test.c @@ -0,0 +1,1063 @@ +/********************************************************************** + Copyright(c) 2022-2023, Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include "utils.h" + +int LLVMFuzzerTestOneInput(const uint8_t *, size_t); +int LLVMFuzzerInitialize(int *, char ***); + +enum ar { +#ifndef __aarch64__ + SSE = 1, + AVX, + AVX2, + AVX512 +#else + AARCH64 = 1, + SVE256 +#endif +}; + +enum ar arch; + +int count = 8; + +static void parse_matched(int argc, char **argv) +{ + int i; + + for (i = 0; i < argc; i++) { +#ifndef __aarch64__ + if (strcmp(argv[i], "SSE") == 0) + arch = SSE; + else if (strcmp(argv[i], "AVX") == 0) + arch = AVX; + else if (strcmp(argv[i], "AVX2") == 0) + arch = AVX2; + else if (strcmp(argv[i], "AVX512") == 0) + arch = AVX512; +#else + if (strcmp(argv[i], "AARCH64") == 0) + arch = AARCH64; + else if (strcmp(argv[i], "SVE256") == 0) + arch = SVE256; +#endif + } +} + +int LLVMFuzzerInitialize(int *argc, char ***argv) +{ + int i; + + for (i = 0; i < *argc; i++) { + /* + * Check if the current argument matches the + * argument we are looking for. + */ + if (strcmp((*argv)[i], "custom") == 0) { + parse_matched(*argc - (i + 1), &((*argv)[i + 1])); + /* + * Remove the matching argument and all arguments + * after it from the command line. + */ + *argc = i; + + break; + } + } + return 0; +} + +static void test_snow3g_init_key_sched(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *init_key = buff; + snow3g_key_schedule_t exp_key_s; + snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + IMB_SNOW3G_INIT_KEY_SCHED(p_mgr, init_key, exp_key); +} + +static void test_snow3g_f8_1_buff_bit(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + + len = len * 8; + const uint8_t *iv = buff; + const uint32_t offset = *(uint32_t *)buff; + + IMB_SNOW3G_F8_1_BUFFER_BIT(p_mgr, exp_key, iv, in, out, len, offset); +} + +static void test_snow3g_f8_1_buff(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + + IMB_SNOW3G_F8_1_BUFFER(p_mgr, exp_key, iv, in, out, len); +} + +static void test_snow3g_f8_2_buff(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + + IMB_SNOW3G_F8_2_BUFFER(p_mgr, exp_key, iv, iv, in, + out, len, in, out, len); +} + +static void test_snow3g_f8_4_buff(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + + IMB_SNOW3G_F8_4_BUFFER(p_mgr, exp_key, iv, iv, iv, + iv, in, out, len, in, out, + len, in, out, len, + in, out, len); +} + +static void test_snow3g_f8_8_buff(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + + IMB_SNOW3G_F8_8_BUFFER(p_mgr, exp_key, iv, iv, iv, + iv, iv, iv, iv, iv, + in, out, len, in, out, len, + in, out, len, + in, out, len, in, out, + len, in, out, len, in, + out, len, in, out, len); +} + +static void test_snow3g_f8_n_buff(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + const void *iv[8]; + const void *in[8]; + void *out[8]; + uint32_t len[8]; + + for (int i = 0; i < 8; i++) { + iv[i] = buff; + in[i] = buff; + out[i] = buff; + len[i] = dataSize; + } + + IMB_SNOW3G_F8_N_BUFFER(p_mgr, exp_key, iv, in, out, len, count); +} + +static void test_snow3g_f8_8_multikey(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + const void *iv[8]; + const void *in[8]; + void *out[8]; + uint32_t len[8]; + + for (int i = 0; i < 8; i++) { + iv[i] = buff; + in[i] = buff; + out[i] = buff; + len[i] = dataSize; + } + + IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(p_mgr, &exp_key, iv, in, out, len); +} + +static void test_snow3g_f8_n_multikey(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + const void *iv[8]; + const void *in[8]; + void *out[8]; + uint32_t len[8]; + + for (int i = 0; i < 8; i++) { + iv[i] = buff; + in[i] = buff; + out[i] = buff; + len[i] = dataSize; + } + + IMB_SNOW3G_F8_N_BUFFER_MULTIKEY(p_mgr, &exp_key, iv, in, out, len, + count); +} + +static void test_snow3g_f9_1_buff(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const snow3g_key_schedule_t exp_key_s; + const snow3g_key_schedule_t *exp_key = &exp_key_s; + + if (dataSize < sizeof(exp_key_s)) + return; + + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + uint8_t *auth_tag = buff; + + IMB_SNOW3G_F9_1_BUFFER(p_mgr, exp_key, iv, in, len, auth_tag); +} + +#ifndef __aarch64__ +static void test_aes128_gcm_enc(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + + IMB_AES128_GCM_ENC(p_mgr, key, ctx, out, in, len, iv, aad, + aad_len, auth_tag, tag_len); +} + +static void test_aes128_gcm_dec(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES128_GCM_DEC(p_mgr, key, ctx, out, in, len, iv, aad, + aad_len, auth_tag, tag_len); +} + +static void test_aes192_gcm_enc(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES192_GCM_ENC(p_mgr, key, ctx, out, in, len, iv, aad, + aad_len, auth_tag, tag_len); +} + +static void test_aes192_gcm_dec(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES192_GCM_DEC(p_mgr, key, ctx, out, in, len, iv, aad, + aad_len, auth_tag, tag_len); +} + +static void test_aes256_gcm_enc(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES256_GCM_ENC(p_mgr, key, ctx, out, in, len, iv, aad, + aad_len, auth_tag, tag_len); +} + +static void test_aes256_gcm_dec(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES256_GCM_DEC(p_mgr, key, ctx, out, in, len, iv, aad, + aad_len, auth_tag, tag_len); +} + +static void test_aes128_gcm_enc_sgl(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES128_GCM_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_AES128_GCM_ENC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_AES128_GCM_ENC_FINALIZE(p_mgr, key, ctx, auth_tag, tag_len); +} + +static void test_aes128_gcm_dec_sgl(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES128_GCM_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_AES128_GCM_DEC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_AES128_GCM_DEC_FINALIZE(p_mgr, key, ctx, auth_tag, tag_len); +} + +static void test_aes192_gcm_enc_sgl(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES192_GCM_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_AES192_GCM_ENC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_AES192_GCM_ENC_FINALIZE(p_mgr, key, ctx, auth_tag, tag_len); +} + +static void test_aes192_gcm_dec_sgl(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES192_GCM_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_AES192_GCM_DEC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_AES192_GCM_DEC_FINALIZE(p_mgr, key, ctx, auth_tag, tag_len); +} + +static void test_aes256_gcm_enc_sgl(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES256_GCM_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_AES256_GCM_ENC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_AES256_GCM_ENC_FINALIZE(p_mgr, key, ctx, auth_tag, tag_len); +} + +static void test_aes256_gcm_dec_sgl(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + if ((dataSize < sizeof(struct gcm_key_data)) || + (dataSize < sizeof(struct gcm_context_data))) + return; + + const struct gcm_key_data *key = (const struct gcm_key_data *)buff; + struct gcm_context_data *ctx = (struct gcm_context_data *)buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = (uint64_t) *buff; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_AES256_GCM_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_AES256_GCM_DEC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_AES256_GCM_DEC_FINALIZE(p_mgr, key, ctx, auth_tag, tag_len); +} +#endif /* __aarch64__ */ + +static void test_zuc_eea3_1_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *key = buff; + + if (dataSize < IMB_ZUC_KEY_LEN_IN_BYTES) + return; + + void *out = buff; + const void *in = buff; + const uint32_t len = dataSize; + const void *iv = (const void *) buff; + + IMB_ZUC_EEA3_1_BUFFER(p_mgr, key, iv, in, out, len); +} + +static void test_zuc_eea3_4_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *key[4]; + + if (dataSize < IMB_ZUC_KEY_LEN_IN_BYTES) + return; + + const void *iv[4]; + const void *in[4]; + void *out[4]; + uint32_t len[4]; + + for (int i = 0; i < 4; i++) { + key[i] = buff; + iv[i] = buff; + in[i] = buff; + out[i] = buff; + len[i] = dataSize; + } + + IMB_ZUC_EEA3_4_BUFFER(p_mgr, key, iv, in, out, len); +} + +static void test_zuc_eea3_n_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *key[8]; + + if (dataSize < IMB_ZUC_KEY_LEN_IN_BYTES) + return; + + const void *iv[8]; + const void *in[8]; + void *out[8]; + uint32_t len[8]; + + for (int i = 0; i < count; i++) { + key[i] = buff; + iv[i] = buff; + in[i] = buff; + out[i] = buff; + len[i] = dataSize; + } + + IMB_ZUC_EEA3_N_BUFFER(p_mgr, key, iv, in, out, len, count); +} + +static void test_zuc_eia3_1_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *key = buff; + + if (dataSize < IMB_ZUC_KEY_LEN_IN_BYTES) + return; + + const void *in = buff; + uint32_t len = dataSize * 8; + const void *iv = (const void *) buff; + uint32_t *tag = (uint32_t *)buff; + + IMB_ZUC_EIA3_1_BUFFER(p_mgr, key, iv, in, len, tag); +} + +static void test_zuc_eia3_n_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *key[8]; + + if (dataSize < IMB_ZUC_KEY_LEN_IN_BYTES) + return; + + const void *iv[8]; + const void *in[8]; + void *tag_ptr_array[8]; + uint32_t len[8]; + uint32_t *tag[8]; + + for (int i = 0; i < count; i++) { + key[i] = buff; + iv[i] = buff; + in[i] = buff; + tag_ptr_array[i] = buff; + tag[i] = (uint32_t *)buff; + len[i] = dataSize * 8; + } + + IMB_ZUC_EIA3_N_BUFFER(p_mgr, key, iv, in, len, tag, count); +} + +#ifndef __aarch64__ +static void test_chacha_poly_enc(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if (dataSize < sizeof(struct chacha20_poly1305_context_data)) + return; + + struct chacha20_poly1305_context_data *ctx = + (struct chacha20_poly1305_context_data *)buff; + const void *key = buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = dataSize; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_CHACHA20_POLY1305_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_CHACHA20_POLY1305_ENC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_CHACHA20_POLY1305_ENC_FINALIZE(p_mgr, ctx, auth_tag, tag_len); +} + +static void test_chacha_poly_dec(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + if (dataSize < sizeof(struct chacha20_poly1305_context_data)) + return; + + struct chacha20_poly1305_context_data *ctx = + (struct chacha20_poly1305_context_data *)buff; + const void *key = buff; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint8_t *iv = buff; + const uint8_t *aad = buff; + uint64_t aad_len = dataSize; + uint8_t *auth_tag = buff; + uint64_t tag_len = (uint64_t) *buff; + + IMB_CHACHA20_POLY1305_INIT(p_mgr, key, ctx, iv, aad, aad_len); + IMB_CHACHA20_POLY1305_DEC_UPDATE(p_mgr, key, ctx, out, in, len); + IMB_CHACHA20_POLY1305_DEC_FINALIZE(p_mgr, ctx, auth_tag, tag_len); +} + +static void test_crc32_ethernet_fcs(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC32_ETHERNET_FCS(p_mgr, in, len); +} + +static void test_crc16_x25(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC16_X25(p_mgr, in, len); +} + +static void test_crc32_sctp(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC32_SCTP(p_mgr, in, len); +} + +static void test_crc24_lte_a(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC24_LTE_A(p_mgr, in, len); +} + +static void test_crc24_lte_b(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC24_LTE_B(p_mgr, in, len); +} + +static void test_crc16_fp_data(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC16_FP_DATA(p_mgr, in, len); +} + +static void test_crc11_fp_header(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC11_FP_HEADER(p_mgr, in, len); +} + +static void test_crc7_fp_header(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC7_FP_HEADER(p_mgr, in, len); +} + +static void test_crc10_iuup_data(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC10_IUUP_DATA(p_mgr, in, len); +} + +static void test_crc6_iuup_header(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC6_IUUP_HEADER(p_mgr, in, len); +} + +static void test_crc32_wimax_ofdma_data(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC32_WIMAX_OFDMA_DATA(p_mgr, in, len); +} + +static void test_crc8_wimax_ofdma_hcs(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *in = buff; + const uint64_t len = dataSize; + + IMB_CRC8_WIMAX_OFDMA_HCS(p_mgr, in, len); +} + +static void test_kasumi_f8_init_key_sched(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *key = buff; + kasumi_key_sched_t exp_key_s; + kasumi_key_sched_t *exp_key = &exp_key_s; + + if (dataSize < IMB_KASUMI_KEY_SIZE) + return; + + IMB_KASUMI_INIT_F8_KEY_SCHED(p_mgr, key, exp_key); +} + +static void test_kasumi_f8_1_buff_bit(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + const uint32_t offset = (uint32_t) *buff * 8; + + if (offset >= (dataSize * 8)) + return; + + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = (dataSize * 8) - offset; + const uint64_t iv = *((uint64_t *) buff); + + IMB_KASUMI_F8_1_BUFFER_BIT(p_mgr, exp_key, iv, in, out, len, offset); +} + +static void test_kasumi_f8_1_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint64_t iv = *((uint64_t *) buff); + + IMB_KASUMI_F8_1_BUFFER(p_mgr, exp_key, iv, in, out, len); +} + +static void test_kasumi_f8_2_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint64_t iv = *((uint64_t *) buff); + + IMB_KASUMI_F8_2_BUFFER(p_mgr, exp_key, iv, iv, in, + out, len, in, out, len); +} + +static void test_kasumi_f8_3_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint64_t iv = *((uint64_t *) buff); + + IMB_KASUMI_F8_3_BUFFER(p_mgr, exp_key, iv, iv, iv, in, out, + in, out, in, out, len); +} + +static void test_kasumi_f8_4_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + uint8_t *out = buff; + const uint8_t *in = buff; + uint64_t len = dataSize; + const uint64_t iv = *((uint64_t *) buff); + + IMB_KASUMI_F8_4_BUFFER(p_mgr, exp_key, iv, iv, iv, + iv, in, out, in, out, + in, out, in, out, len); +} + +static void test_kasumi_f8_n_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + const uint64_t *iv = (uint64_t *) buff; + const void *in[8]; + void *out[8]; + uint32_t len[8]; + + for (int i = 0; i < count; i++) { + in[i] = buff; + out[i] = buff; + len[i] = dataSize; + } + + IMB_KASUMI_F8_N_BUFFER(p_mgr, exp_key, iv, in, out, len, count); +} + +static void test_kasumi_f9_1_buff(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + const uint8_t *in = buff; + uint64_t len = dataSize; + uint8_t *tag = buff; + + IMB_KASUMI_F9_1_BUFFER(p_mgr, exp_key, in, len, tag); +} + +static void test_kasumi_f9_1_buff_user(IMB_MGR *p_mgr, uint8_t *buff, size_t dataSize) +{ + const kasumi_key_sched_t exp_key_s; + const kasumi_key_sched_t *exp_key = &exp_key_s; + const uint8_t *in = buff; + uint64_t len = dataSize * 8; + uint8_t *tag = buff; + const uint64_t iv = (uint64_t) buff; + const uint32_t dir = (uint32_t) *buff * 8; + + IMB_KASUMI_F9_1_BUFFER_USER(p_mgr, exp_key, iv, in, len, tag, dir); +} + +static void test_kasumi_f9_init_key_sched(IMB_MGR *p_mgr, uint8_t *buff, + size_t dataSize) +{ + const void *key = buff; + kasumi_key_sched_t exp_key_s; + kasumi_key_sched_t *exp_key = &exp_key_s; + + if (dataSize < IMB_KASUMI_KEY_SIZE) + return; + + IMB_KASUMI_INIT_F9_KEY_SCHED(p_mgr, key, exp_key); +} +#endif /* __aarch64__ */ + +struct { + void (*func)(IMB_MGR *mb_mgr, uint8_t *buff, size_t dataSize); + const char *func_name; +} direct_apis[] = { + {test_snow3g_init_key_sched, "test_snow3g_init_key_sched"}, + {test_snow3g_f8_1_buff_bit, "test_snow3g_f8_1_buff_bit"}, + {test_snow3g_f8_1_buff, "test_snow3g_f8_1_buff"}, + {test_snow3g_f8_2_buff, "test_snow3g_f8_2_buff"}, + {test_snow3g_f8_4_buff, "test_snow3g_f8_4_buff"}, + {test_snow3g_f8_8_buff, "test_snow3g_f8_8_buff"}, + {test_snow3g_f8_n_buff, "test_snow3g_f8_n_buff"}, + {test_snow3g_f8_8_multikey, "test_snow3g_f8_8_multikey"}, + {test_snow3g_f8_n_multikey, "test_snow3g_f8_n_multikey"}, + {test_snow3g_f9_1_buff, "test_snow3g_f9_1_buff"}, +#ifndef __aarch64__ + {test_aes128_gcm_enc_sgl, "test_aes128_gcm_enc_sgl"}, + {test_aes128_gcm_dec_sgl, "test_aes128_gcm_dec_sgl"}, + {test_aes192_gcm_enc_sgl, "test_aes192_gcm_enc_sgl"}, + {test_aes192_gcm_dec_sgl, "test_aes192_gcm_dec_sgl"}, + {test_aes256_gcm_enc_sgl, "test_aes256_gcm_enc_sgl"}, + {test_aes256_gcm_dec_sgl, "test_aes256_gcm_dec_sgl"}, + {test_aes128_gcm_enc, "test_aes128_gcm_enc"}, + {test_aes128_gcm_dec, "test_aes128_gcm_dec"}, + {test_aes192_gcm_enc, "test_aes192_gcm_enc"}, + {test_aes192_gcm_dec, "test_aes192_gcm_dec"}, + {test_aes256_gcm_enc, "test_aes256_gcm_enc"}, + {test_aes256_gcm_dec, "test_aes256_gcm_dec"}, +#endif /* __aarch64__ */ + {test_zuc_eea3_1_buff, "test_zuc_eea3_1_buff"}, + {test_zuc_eea3_4_buff, "test_zuc_eea3_4_buff"}, + {test_zuc_eea3_n_buff, "test_zuc_eea3_n_buff"}, + {test_zuc_eia3_1_buff, "test_zuc_eia3_1_buff"}, + {test_zuc_eia3_n_buff, "test_zuc_eia3_n_buff"}, +#ifndef __aarch64__ + {test_chacha_poly_enc, "test_chacha_poly_enc"}, + {test_chacha_poly_dec, "test_chacha_poly_dec"}, + {test_crc32_ethernet_fcs, "test_crc32_ethernet_fcs"}, + {test_crc16_x25, "test_crc16_x25"}, + {test_crc32_sctp, "test_crc32_sctp"}, + {test_crc16_fp_data, "test_crc16_fp_data"}, + {test_crc11_fp_header, "test_crc11_fp_header"}, + {test_crc24_lte_a, "test_crc24_lte_a"}, + {test_crc24_lte_b, "test_crc24_lte_b"}, + {test_crc7_fp_header, "test_crc7_fp_header"}, + {test_crc10_iuup_data, "test_crc10_iuup_data"}, + {test_crc6_iuup_header, "test_crc6_iuup_header"}, + {test_crc32_wimax_ofdma_data, "test_crc32_wimax_ofdma_data"}, + {test_crc8_wimax_ofdma_hcs, "test_crc8_wimax_ofdma_hcs"}, + {test_kasumi_f8_init_key_sched, "test_kasumi_f8_init_key_sched"}, + {test_kasumi_f8_1_buff_bit, "test_kasumi_f8_1_buff_bit"}, + {test_kasumi_f8_1_buff, "test_kasumi_f8_1_buff"}, + {test_kasumi_f8_2_buff, "test_kasumi_f8_2_buff"}, + {test_kasumi_f8_3_buff, "test_kasumi_f8_3_buff"}, + {test_kasumi_f8_4_buff, "test_kasumi_f8_4_buff"}, + {test_kasumi_f8_n_buff, "test_kasumi_f8_n_buff"}, + {test_kasumi_f9_1_buff, "test_kasumi_f9_1_buff"}, + {test_kasumi_f9_1_buff_user, "test_kasumi_f9_1_buff_user"}, + {test_kasumi_f9_init_key_sched, "test_kasumi_f9_init_key_sched"}, +#endif /* __aarch64__ */ +}; + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) +{ + IMB_ARCH arch_to_run = IMB_ARCH_NUM; + static IMB_MGR *p_mgr = NULL; + uint8_t *buff; + + buff = malloc(dataSize); + if (buff == NULL) + return EXIT_FAILURE; + memcpy(buff, data, dataSize); + + /* allocate multi-buffer manager */ + if (p_mgr == NULL) { + p_mgr = alloc_mb_mgr(0); + if (p_mgr == NULL) { + printf("Error allocating MB_MGR structure!\n"); + free(buff); + return EXIT_FAILURE; + } + +#ifndef __aarch64__ + if (arch == SSE) + init_mb_mgr_sse(p_mgr); + else if (arch == AVX) + init_mb_mgr_avx(p_mgr); + else if (arch == AVX2) + init_mb_mgr_avx2(p_mgr); + else if (arch == AVX512) + init_mb_mgr_avx512(p_mgr); +#else /* __aarch64__ */ + if (arch == AARCH64) + init_mb_mgr_aarch64(p_mgr); + else if (arch == SVE256) + init_mb_mgr_aarch64_sve256(p_mgr); +#endif /* __aarch64__ */ + else + init_mb_mgr_auto(p_mgr, &arch_to_run); + } + + const int idx = data[0]%DIM(direct_apis); + + direct_apis[idx].func(p_mgr, buff, dataSize); + + free(buff); + return 0; +} diff --git a/test/job_api_fuzz_test.c b/test/fuzz-app/job_api_fuzz_test.c similarity index 96% rename from test/job_api_fuzz_test.c rename to test/fuzz-app/job_api_fuzz_test.c index 0fdf71921bc9b8997c349e807f4b0505a2bba0ad..7f6f7587797e0dc38d7cb7f80aabfa8e5b2dbdb4 100644 --- a/test/job_api_fuzz_test.c +++ b/test/fuzz-app/job_api_fuzz_test.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2021-2022, Intel Corporation All rights reserved. + Copyright(c) 2021-2023, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include #define BUFF_SIZE (32*1024*1024) +#define MAX_BURST_JOBS 32 int LLVMFuzzerTestOneInput(const uint8_t *, size_t); @@ -478,7 +479,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) if (dataSize < sizeof(IMB_JOB)) return 0; - if (num_jobs > 32 || num_jobs == 0 || key_len == 0) + if (num_jobs > MAX_BURST_JOBS || num_jobs == 0 || key_len == 0) return 0; if (cipher_dir != NULL) { @@ -506,25 +507,22 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) #ifdef __aarch64__ if (strcmp(ar, "aarch64") == 0) init_mb_mgr_aarch64(p_mgr); -#endif /* aarch64 */ - -#ifdef __x86_64__ - if (strcmp(ar, "avx") == 0) +#else /* __aarch64__ */ + if (strcasecmp(ar, "AVX") == 0) init_mb_mgr_avx(p_mgr); - else if (strcmp(ar, "avx2") == 0) + else if (strcasecmp(ar, "AVX2") == 0) init_mb_mgr_avx2(p_mgr); - else if (strcmp(ar, "avx512") == 0) + else if (strcasecmp(ar, "AVX512") == 0) init_mb_mgr_avx512(p_mgr); - else if (strcmp(ar, "sse") == 0) + else if (strcasecmp(ar, "SSE") == 0) init_mb_mgr_sse(p_mgr); -#endif /* x86_64 */ +#endif /* __aarch64__ */ else init_mb_mgr_auto(p_mgr, &arch); } IMB_JOB *job = NULL; /* create job array */ - IMB_JOB *jobs[32] = {NULL}; if (api == NULL || (strcmp(api, "SINGLE") == 0)) { single = true; @@ -567,6 +565,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) job = IMB_SUBMIT_JOB(p_mgr); } } else if (burst) { + IMB_JOB *jobs[MAX_BURST_JOBS] = {NULL}; + while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) < (uint32_t)num_jobs) IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); @@ -604,12 +604,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_SUBMIT_BURST(p_mgr, num_jobs, jobs); } else if (cipher_burst) { - while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) - < (uint32_t)num_jobs) - IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); + IMB_JOB jobs[MAX_BURST_JOBS] = {0}; for (i = 0; i < num_jobs; i++) { - job = jobs[i]; + job = &jobs[i]; cipher = cipher_selection(); memcpy(job, data, sizeof(*job)); /* @@ -626,17 +624,16 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) (uint8_t buff[2*BUFF_SIZE], 64); fill_job_data(job, buff); + fill_additional_cipher_data(job, buff, buffsize); } - IMB_SUBMIT_CIPHER_BURST(p_mgr, job, num_jobs, + IMB_SUBMIT_CIPHER_BURST(p_mgr, jobs, num_jobs, cipher, dir, key_len); } else if (hash_burst) { - while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) - < (uint32_t)num_jobs) - IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); + IMB_JOB jobs[MAX_BURST_JOBS] = {0}; for (i = 0; i < num_jobs; i++) { - job = jobs[i]; + job = &jobs[i]; hash = hash_selection(); memcpy(job, data, sizeof(*job)); /* @@ -653,9 +650,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) (uint8_t buff[2*BUFF_SIZE], 64); fill_job_data(job, buff); + fill_additional_hash_data(job, buff, buffsize); } - IMB_SUBMIT_HASH_BURST(p_mgr, jobs[0], num_jobs, hash); + IMB_SUBMIT_HASH_BURST(p_mgr, jobs, num_jobs, hash); } free_mb_mgr(p_mgr); diff --git a/test/hmac_md5_test.c b/test/hmac_md5_test.c deleted file mode 100644 index b95e87bb5d30ab23a2540bc0a4ea8d13a41dbc35..0000000000000000000000000000000000000000 --- a/test/hmac_md5_test.c +++ /dev/null @@ -1,546 +0,0 @@ -/***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include -#include -#include - -#include -#include "gcm_ctr_vectors_test.h" -#include "utils.h" - -int hmac_md5_test(struct IMB_MGR *mb_mgr); - -#define block_size 64 -#define digest_size 16 -#define digest96_size 12 - -/* - * Test vectors from https://tools.ietf.org/html/rfc2202 - */ - -/* - * 2. Test Case 1 - * - * Key = 0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b - * - * Key length = 16 - * - * Data = "Hi There" - * - * Data length = 8 - * - * Digest = 0x9294727a3638bb1c13f48ef8158bfc9d - * - * Digest96 = 0x9294727a3638bb1c13f48ef8 - */ -#define test_case1 "1" -#define test_case_l1 "1_long" -#define key_len1 16 -#define data_len1 8 -#define digest_len1 digest96_size -#define digest_len_l1 digest_size -static const uint8_t key1[key_len1] = { - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b -}; -static const char data1[] = "Hi There"; -static const uint8_t digest1[digest_len_l1] = { - 0x92, 0x94, 0x72, 0x7a, 0x36, 0x38, 0xbb, 0x1c, - 0x13, 0xf4, 0x8e, 0xf8, 0x15, 0x8b, 0xfc, 0x9d -}; - -/* - * 2. Test Case 2 - * - * Key = "Jefe" - * - * Key length = 4 - * - * Data = "what do ya want for nothing?" - * - * Data length = 28 - * - * Digest = 0x750c783e6ab0b503eaa86e310a5db738 - * - * Digest96 = 0x750c783e6ab0b503eaa86e31 - */ -#define test_case2 "2" -#define test_case_l2 "2_long" -#define key_len2 4 -#define data_len2 28 -#define digest_len2 digest96_size -#define digest_len_l2 digest_size -static const char key2[] = "Jefe"; -static const char data2[] = "what do ya want for nothing?"; -static const uint8_t digest2[digest_len_l2] = { - 0x75, 0x0c, 0x78, 0x3e, 0x6a, 0xb0, 0xb5, 0x03, - 0xea, 0xa8, 0x6e, 0x31, 0x0a, 0x5d, 0xb7, 0x38 -}; - -/* - * 2. Test Case 3 - * - * Key = 0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * - * Key length = 16 - * - * Data = 0xdd (repeated 50 times) - * - * Data length = 50 - * - * Digest = 0x56be34521d144c88dbb8c733f0e8b3f6 - * - * Digest96 = 0x56be34521d144c88dbb8c733 - */ -#define test_case3 "3" -#define test_case_l3 "3_long" -#define key_len3 16 -#define data_len3 50 -#define digest_len3 digest96_size -#define digest_len_l3 digest_size -static const uint8_t key3[key_len3] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa -}; -static const uint8_t data3[data_len3] = { - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd -}; -static const uint8_t digest3[digest_len_l3] = { - 0x56, 0xbe, 0x34, 0x52, 0x1d, 0x14, 0x4c, 0x88, - 0xdb, 0xb8, 0xc7, 0x33, 0xf0, 0xe8, 0xb3, 0xf6 -}; - -/* - * 2. Test Case 4 - * - * Key = 0x0102030405060708090a0b0c0d0e0f10111213141516171819 - * - * Key length = 25 - * - * Data = 0xcd (repeated 50 times) - * - * Data length = 50 - * - * Digest = 0x697eaf0aca3a3aea3a75164746ffaa79 - * - * Digest96 = 0x697eaf0aca3a3aea3a751647 - */ -#define test_case4 "4" -#define test_case_l4 "4_long" -#define key_len4 25 -#define data_len4 50 -#define digest_len4 digest96_size -#define digest_len_l4 digest_size -static const uint8_t key4[key_len4] = { - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, - 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, - 0x19 -}; -static const uint8_t data4[data_len4] = { - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd -}; -static const uint8_t digest4[digest_len_l4] = { - 0x69, 0x7e, 0xaf, 0x0a, 0xca, 0x3a, 0x3a, 0xea, - 0x3a, 0x75, 0x16, 0x47, 0x46, 0xff, 0xaa, 0x79 -}; - -/* - * 2. Test Case 5 - * - * Key = 0x0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c - * - * Key length = 16 - * - * Data = "Test With Truncation" - * - * Data length = 20 - * - * Digest = 0x56461ef2342edc00f9bab995690efd4c - * - * Digest96 = 0x56461ef2342edc00f9bab995 - */ -#define test_case5 "5" -#define test_case_l5 "5_long" -#define key_len5 16 -#define data_len5 20 -#define digest_len5 digest96_size -#define digest_len_l5 digest_size -static const uint8_t key5[key_len5] = { - 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, - 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c -}; -static const char data5[] = "Test With Truncation"; -static const uint8_t digest5[digest_len_l5] = { - 0x56, 0x46, 0x1e, 0xf2, 0x34, 0x2e, 0xdc, 0x00, - 0xf9, 0xba, 0xb9, 0x95, 0x69, 0x0e, 0xfd, 0x4c -}; - -/* - * 2. Test Case 6 - * - * Key = 0xaa (repeated 80 times) - * - * Key length = 80 - * - * Data = "Test Using Larger Than Block-Size Key - Hash Key First" - * - * Data length = 54 - * - * Digest = 0x6b1ab7fe4bd7bf8f0b62e6ce61b9d0cd - * - * Digest96 = 0x6b1ab7fe4bd7bf8f0b62e6ce - */ -/* #define test_case6 "6" */ -/* #define key_len6 80 */ -/* #define data_len6 54 */ -/* #define digest_len6 digest96_size */ -/* static const uint8_t key6[key_len6] = { */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa */ -/* }; */ -/* static const char data6[] = "Test Using Larger Than Block-Size " */ -/* "Key - Hash Key First"; */ -/* static const uint8_t digest6[digest_len6] = { */ -/* 0x6b, 0x1a, 0xb7, 0xfe, 0x4b, 0xd7, 0xbf, 0x8f, */ -/* 0x0b, 0x62, 0xe6, 0xce */ -/* }; */ - -/* - * 2. Test Case 7 - * - * Key = 0xaa (repeated 80 times) - * - * Key length = 80 - * - * Data = "Test Using Larger Than Block-Size Key and Larger" - * - * Data length = 73 - * - * Digest = 0x6f630fad67cda0ee1fb1f562db3aa53e - * - * Digest96 = 0x6f630fad67cda0ee1fb1f562 - */ -/* #define test_case7 "7" */ -/* #define key_len7 80 */ -/* #define data_len7 73 */ -/* #define digest_len7 digest96_size */ -/* static const uint8_t key7[key_len7] = { */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, */ -/* 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa */ -/* }; */ -/* static const char data7[] = "Test Using Larger Than Block-Size " */ -/* "Key and Larger Than One Block-Size Data"; */ -/* static const uint8_t digest7[digest_len7] = { */ -/* 0x6f, 0x63, 0x0f, 0xad, 0x67, 0xcd, 0xa0, 0xee, */ -/* 0x1f, 0xb1, 0xf5, 0x62 */ -/* }; */ - -#define HMAC_MD5_TEST_VEC(num) \ - { test_case##num, \ - (const uint8_t *) key##num, key_len##num, \ - (const uint8_t *) data##num, data_len##num, \ - (const uint8_t *) digest##num, digest_len##num } -#define HMAC_MD5_TEST_VEC_LONG(num) \ - { test_case_l##num, \ - (const uint8_t *) key##num, key_len##num, \ - (const uint8_t *) data##num, data_len##num, \ - (const uint8_t *) digest##num, digest_len_l##num } - -static const struct hmac_md5_rfc2202_vector { - const char *test_case; - const uint8_t *key; - size_t key_len; - const uint8_t *data; - size_t data_len; - const uint8_t *digest; - size_t digest_len; -} hmac_md5_vectors[] = { - HMAC_MD5_TEST_VEC(1), - HMAC_MD5_TEST_VEC(2), - HMAC_MD5_TEST_VEC(3), - HMAC_MD5_TEST_VEC(4), - HMAC_MD5_TEST_VEC(5), - /* HMAC_MD5_TEST_VEC(6), */ - /* HMAC_MD5_TEST_VEC(7), */ - HMAC_MD5_TEST_VEC_LONG(1), - HMAC_MD5_TEST_VEC_LONG(2), - HMAC_MD5_TEST_VEC_LONG(3), - HMAC_MD5_TEST_VEC_LONG(4), - HMAC_MD5_TEST_VEC_LONG(5), -}; - -static int -hmac_md5_job_ok(const struct hmac_md5_rfc2202_vector *vec, - const struct IMB_JOB *job, - const uint8_t *auth, - const uint8_t *padding, - const size_t sizeof_padding) -{ - if (job->status != IMB_STATUS_COMPLETED) { - printf("line:%d job error status:%d ", __LINE__, job->status); - return 0; - } - - /* hash checks */ - if (memcmp(padding, &auth[sizeof_padding + vec->digest_len], - sizeof_padding)) { - printf("hash overwrite tail\n"); - hexdump(stderr, "Target", - &auth[sizeof_padding + vec->digest_len], - sizeof_padding); - return 0; - } - - if (memcmp(padding, &auth[0], sizeof_padding)) { - printf("hash overwrite head\n"); - hexdump(stderr, "Target", &auth[0], sizeof_padding); - return 0; - } - - if (memcmp(vec->digest, &auth[sizeof_padding], - vec->digest_len)) { - printf("hash mismatched\n"); - hexdump(stderr, "Received", &auth[sizeof_padding], - vec->digest_len); - hexdump(stderr, "Expected", vec->digest, - vec->digest_len); - return 0; - } - return 1; -} - -static int -test_hmac_md5(struct IMB_MGR *mb_mgr, - const struct hmac_md5_rfc2202_vector *vec, - const int num_jobs) -{ - struct IMB_JOB *job; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - int i = 0, jobs_rx = 0, ret = -1; - uint8_t key[block_size]; - uint8_t buf[block_size]; - DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); - DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); - int key_len = 0; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - vec->digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (vec->key_len <= block_size) { - memcpy(key, vec->key, vec->key_len); - key_len = (int) vec->key_len; - } else { - printf("Key length longer than block size is not supported " - "by MD5\n"); - ret = 0; - goto end; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_MD5_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_MD5_ONE_BLOCK(mb_mgr, buf, opad_hash); - - /* empty the manager */ - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - job->enc_keys = NULL; - job->dec_keys = NULL; - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->dst = NULL; - job->key_len_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->digest_len; - job->iv = NULL; - job->iv_len_in_bytes = 0; - job->src = vec->data; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = 0; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; - job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_MD5; - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job) { - jobs_rx++; - /* - * HMAC-MD5 requires 8 submissions to get one back - */ - if (num_jobs < 8) { - printf("%d Unexpected return from submit_job\n", - __LINE__); - goto end; - } - if (!hmac_md5_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - jobs_rx++; - if (!hmac_md5_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - - if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); - goto end; - } - ret = 0; - - end: - /* empty the manager before next tests */ - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static void -test_hmac_md5_std_vectors(struct IMB_MGR *mb_mgr, - const int num_jobs, - struct test_suite_context *ts) -{ - const int vectors_cnt = DIM(hmac_md5_vectors); - int vect; - - printf("HMAC-MD5 standard test vectors (N jobs = %d):\n", num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("[%d/%d] RFC2202 Test Case %s key_len:%d data_len:%d " - "digest_len:%d\n", - vect, vectors_cnt, - hmac_md5_vectors[idx].test_case, - (int) hmac_md5_vectors[idx].key_len, - (int) hmac_md5_vectors[idx].data_len, - (int) hmac_md5_vectors[idx].digest_len); -#else - printf("."); -#endif - - if (test_hmac_md5(mb_mgr, &hmac_md5_vectors[idx], num_jobs)) { - printf("error #%d\n", vect); - test_suite_update(ts, 0, 1); - } else { - test_suite_update(ts, 1, 0); - } - } - printf("\n"); -} - -int -hmac_md5_test(struct IMB_MGR *mb_mgr) -{ - struct test_suite_context ts; - int num_jobs, errors = 0; - - test_suite_start(&ts, "HMAC-MD5"); - for (num_jobs = 1; num_jobs <= 17; num_jobs++) - test_hmac_md5_std_vectors(mb_mgr, num_jobs, &ts); - errors = test_suite_end(&ts); - - return errors; -} diff --git a/test/hmac_sha1_test.c b/test/hmac_sha1_test.c deleted file mode 100644 index 8fe7cf1a1ffe3acf5ce95ffb2ec4fc6d7bf63ef6..0000000000000000000000000000000000000000 --- a/test/hmac_sha1_test.c +++ /dev/null @@ -1,812 +0,0 @@ -/***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include -#include -#include - -#include -#include "gcm_ctr_vectors_test.h" -#include "utils.h" - -int hmac_sha1_test(struct IMB_MGR *mb_mgr); - -#define block_size 64 -#define digest_size 20 -#define digest96_size 12 -#define max_burst_jobs 32 - -/* - * Test vectors from https://tools.ietf.org/html/rfc2202 - */ - -/* - * test_case = 1 - * key = 0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b - * key_len = 20 - * data = "Hi There" - * data_len = 8 - * digest = 0xb617318655057264e28bc0b6fb378c8ef146be00 - */ -#define test_case1 "1" -#define key_len1 20 -#define data_len1 8 -#define digest_len1 digest_size -static const uint8_t key1[key_len1] = { - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b -}; -static const char data1[] = "Hi There"; -static const uint8_t digest1[digest_len1] = { - 0xb6, 0x17, 0x31, 0x86, 0x55, 0x05, 0x72, 0x64, - 0xe2, 0x8b, 0xc0, 0xb6, 0xfb, 0x37, 0x8c, 0x8e, - 0xf1, 0x46, 0xbe, 0x00 -}; - -/* - * test_case = 2 - * key = "Jefe" - * key_len = 4 - * data = "what do ya want for nothing?" - * data_len = 28 - * digest = 0xeffcdf6ae5eb2fa2d27416d5f184df9c259a7c79 - */ -#define test_case2 "2" -#define key_len2 4 -#define data_len2 28 -#define digest_len2 digest_size -static const char key2[] = "Jefe"; -static const char data2[] = "what do ya want for nothing?"; -static const uint8_t digest2[digest_len2] = { - 0xef, 0xfc, 0xdf, 0x6a, 0xe5, 0xeb, 0x2f, 0xa2, - 0xd2, 0x74, 0x16, 0xd5, 0xf1, 0x84, 0xdf, 0x9c, - 0x25, 0x9a, 0x7c, 0x79 -}; - -/* - * test_case = 3 - * key = 0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * key_len = 20 - * data = 0xdd repeated 50 times - * data_len = 50 - * digest = 0x125d7342b9ac11cd91a39af48aa17b4f63f175d3 - */ -#define test_case3 "3" -#define key_len3 20 -#define data_len3 50 -#define digest_len3 digest_size -static const uint8_t key3[key_len3] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa -}; -static const uint8_t data3[data_len3] = { - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd -}; -static const uint8_t digest3[digest_len3] = { - 0x12, 0x5d, 0x73, 0x42, 0xb9, 0xac, 0x11, 0xcd, - 0x91, 0xa3, 0x9a, 0xf4, 0x8a, 0xa1, 0x7b, 0x4f, - 0x63, 0xf1, 0x75, 0xd3 -}; - -/* - * test_case = 4 - * key = 0x0102030405060708090a0b0c0d0e0f10111213141516171819 - * key_len = 25 - * data = 0xcd repeated 50 times - * data_len = 50 - * digest = 0x4c9007f4026250c6bc8414f9bf50c86c2d7235da - */ -#define test_case4 "4" -#define key_len4 25 -#define data_len4 50 -#define digest_len4 digest_size -static const uint8_t key4[key_len4] = { - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, - 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, - 0x19 -}; -static const uint8_t data4[data_len4] = { - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd -}; -static const uint8_t digest4[digest_len4] = { - 0x4c, 0x90, 0x07, 0xf4, 0x02, 0x62, 0x50, 0xc6, - 0xbc, 0x84, 0x14, 0xf9, 0xbf, 0x50, 0xc8, 0x6c, - 0x2d, 0x72, 0x35, 0xda -}; - -/* - * test_case = 5 - * key = 0x0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c - * key_len = 20 - * data = "Test With Truncation" - * data_len = 20 - * digest = 0x4c1a03424b55e07fe7f27be1d58bb9324a9a5a04 - * digest-96 = 0x4c1a03424b55e07fe7f27be1 - */ -#define test_case5 "5" -#define key_len5 20 -#define data_len5 20 -#define digest_len5 digest_size -static const uint8_t key5[key_len5] = { - 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, - 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, - 0x0c, 0x0c, 0x0c, 0x0c -}; -static const char data5[] = "Test With Truncation"; -static const uint8_t digest5[digest_len5] = { - 0x4c, 0x1a, 0x03, 0x42, 0x4b, 0x55, 0xe0, 0x7f, - 0xe7, 0xf2, 0x7b, 0xe1, 0xd5, 0x8b, 0xb9, 0x32, - 0x4a, 0x9a, 0x5a, 0x04 -}; - -#define test_case5_96 "5-96" -#define key_len5_96 key_len5 -#define data_len5_96 data_len5 -#define digest_len5_96 digest96_size -#define key5_96 key5 -#define data5_96 data5 -static const uint8_t digest5_96[digest_len5_96] = { - 0x4c, 0x1a, 0x03, 0x42, 0x4b, 0x55, 0xe0, 0x7f, - 0xe7, 0xf2, 0x7b, 0xe1 -}; - -/* - * test_case = 6 - * key = 0xaa repeated 80 times - * key_len = 80 - * data = "Test Using Larger Than Block-Size Key - Hash Key First" - * data_len = 54 - * digest = 0xaa4ae5e15272d00e95705637ce8a3b55ed402112 - */ -#define test_case6 "6" -#define key_len6 80 -#define data_len6 54 -#define digest_len6 digest_size -static const uint8_t key6[key_len6] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, -}; -static const char data6[] = - "Test Using Larger Than Block-Size Key - Hash Key First"; -static const uint8_t digest6[digest_len6] = { - 0xaa, 0x4a, 0xe5, 0xe1, 0x52, 0x72, 0xd0, 0x0e, - 0x95, 0x70, 0x56, 0x37, 0xce, 0x8a, 0x3b, 0x55, - 0xed, 0x40, 0x21, 0x12 -}; - -/* - * test_case = 7 - * key = 0xaa repeated 80 times - * key_len = 80 - * data = "Test Using Larger Than Block-Size Key and Larger - * Than One Block-Size Data" - * data_len = 73 - * digest = 0xe8e99d0f45237d786d6bbaa7965c7808bbff1a91 - */ -#define test_case7 "7" -#define key_len7 80 -#define data_len7 73 -#define digest_len7 digest_size -static const uint8_t key7[key_len7] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, -}; -static const char data7[] = - "Test Using Larger Than Block-Size Key and " - "Larger Than One Block-Size Data"; -static const uint8_t digest7[digest_len7] = { - 0xe8, 0xe9, 0x9d, 0x0f, 0x45, 0x23, 0x7d, 0x78, - 0x6d, 0x6b, 0xba, 0xa7, 0x96, 0x5c, 0x78, 0x08, - 0xbb, 0xff, 0x1a, 0x91 -}; - -/* - * Test vector from https://csrc.nist.gov/csrc/media/publications/fips/198/ - * archive/2002-03-06/documents/fips-198a.pdf - */ -#define test_case8 "8" -#define key_len8 49 -#define data_len8 9 -#define digest_len8 digest96_size -static const uint8_t key8[key_len8] = { - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0 -}; -static const char data8[] = "Sample #4"; -static const uint8_t digest8[digest_len8] = { - 0x9e, 0xa8, 0x86, 0xef, 0xe2, 0x68, 0xdb, 0xec, - 0xce, 0x42, 0x0c, 0x75 -}; - -#define HMAC_SHA1_TEST_VEC(num) \ - { test_case##num, \ - (const uint8_t *) key##num, key_len##num, \ - (const uint8_t *) data##num, data_len##num, \ - (const uint8_t *) digest##num, digest_len##num } - -static const struct hmac_sha1_rfc2202_vector { - const char *test_case; - const uint8_t *key; - size_t key_len; - const uint8_t *data; - size_t data_len; - const uint8_t *digest; - size_t digest_len; -} hmac_sha1_vectors[] = { - HMAC_SHA1_TEST_VEC(1), - HMAC_SHA1_TEST_VEC(2), - HMAC_SHA1_TEST_VEC(3), - HMAC_SHA1_TEST_VEC(4), - HMAC_SHA1_TEST_VEC(5), - HMAC_SHA1_TEST_VEC(5_96), - HMAC_SHA1_TEST_VEC(6), - HMAC_SHA1_TEST_VEC(7), - HMAC_SHA1_TEST_VEC(8) -}; - -static int -hmac_sha1_job_ok(const struct hmac_sha1_rfc2202_vector *vec, - const struct IMB_JOB *job, - const uint8_t *auth, - const uint8_t *padding, - const size_t sizeof_padding) -{ - if (job->status != IMB_STATUS_COMPLETED) { - printf("line:%d job error status:%d ", __LINE__, job->status); - return 0; - } - - /* hash checks */ - if (memcmp(padding, &auth[sizeof_padding + vec->digest_len], - sizeof_padding)) { - printf("hash overwrite tail\n"); - hexdump(stderr, "Target", - &auth[sizeof_padding + vec->digest_len], - sizeof_padding); - return 0; - } - - if (memcmp(padding, &auth[0], sizeof_padding)) { - printf("hash overwrite head\n"); - hexdump(stderr, "Target", &auth[0], sizeof_padding); - return 0; - } - - if (memcmp(vec->digest, &auth[sizeof_padding], - vec->digest_len)) { - printf("hash mismatched\n"); - hexdump(stderr, "Received", &auth[sizeof_padding], - vec->digest_len); - hexdump(stderr, "Expected", vec->digest, - vec->digest_len); - return 0; - } - return 1; -} - -static int -test_hmac_sha1(struct IMB_MGR *mb_mgr, - const struct hmac_sha1_rfc2202_vector *vec, - const uint32_t num_jobs) -{ - struct IMB_JOB *job; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - uint32_t i = 0, jobs_rx = 0; - int ret = -1; - uint8_t key[block_size]; - uint8_t buf[block_size]; - DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); - DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); - uint32_t key_len = 0; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - vec->digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (vec->key_len <= block_size) { - memcpy(key, vec->key, vec->key_len); - key_len = (int) vec->key_len; - } else { - IMB_SHA1(mb_mgr, vec->key, vec->key_len, key); - key_len = digest_size; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); - - /* empty the manager */ - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - job->enc_keys = NULL; - job->dec_keys = NULL; - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->dst = NULL; - job->key_len_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->digest_len; - job->iv = NULL; - job->iv_len_in_bytes = 0; - job->src = vec->data; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = 0; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; - job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_HMAC_SHA_1; - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job) { - jobs_rx++; - /* - * SHANI HMAC-SHA implementation can return a completed - * job after 2nd submission - */ - if (num_jobs < 2) { - printf("%d Unexpected return from submit_job\n", - __LINE__); - goto end; - } - if (!hmac_sha1_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - jobs_rx++; - if (!hmac_sha1_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - - if (jobs_rx != num_jobs) { - printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); - goto end; - } - ret = 0; - - end: - /* empty the manager before next tests */ - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static int -test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, - const struct hmac_sha1_rfc2202_vector *vec, - const uint32_t num_jobs) -{ - struct IMB_JOB *job, *jobs[max_burst_jobs] = {NULL}; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - uint32_t i = 0, jobs_rx = 0; - int ret = -1, err; - uint8_t key[block_size]; - uint8_t buf[block_size]; - DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); - DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); - uint32_t completed_jobs = 0, key_len = 0; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - vec->digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (vec->key_len <= block_size) { - memcpy(key, vec->key, vec->key_len); - key_len = (int) vec->key_len; - } else { - IMB_SHA1(mb_mgr, vec->key, vec->key_len, key); - key_len = digest_size; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); - - while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) - IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); - - for (i = 0; i < num_jobs; i++) { - job = jobs[i]; - job->enc_keys = NULL; - job->dec_keys = NULL; - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->dst = NULL; - job->key_len_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->digest_len; - job->iv = NULL; - job->iv_len_in_bytes = 0; - job->src = vec->data; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = 0; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; - job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_HMAC_SHA_1; - - job->user_data = auths[i]; - - } - - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); - err = imb_get_errno(mb_mgr); - - if (err != 0) { - printf("submit_burst error %d : '%s'\n", err, - imb_get_strerror(err)); - goto end; - } - -check_burst_jobs: - for (i = 0; i < completed_jobs; i++) { - job = jobs[i]; - - if (job->status != IMB_STATUS_COMPLETED) { - printf("job %u status not complete!\n", i+1); - goto end; - } - - if (!hmac_sha1_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - jobs_rx++; - } - - if (jobs_rx != num_jobs) { - completed_jobs = IMB_FLUSH_BURST(mb_mgr, - num_jobs - completed_jobs, - jobs); - if (completed_jobs == 0) { - printf("Expected %u jobs, received %u\n", - num_jobs, jobs_rx); - goto end; - } - goto check_burst_jobs; - } - ret = 0; - - end: - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static int -test_hmac_sha1_hash_burst(struct IMB_MGR *mb_mgr, - const struct hmac_sha1_rfc2202_vector *vec, - const uint32_t num_jobs) -{ - struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - uint32_t i = 0, jobs_rx = 0; - int ret = -1; - uint8_t key[block_size]; - uint8_t buf[block_size]; - DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); - DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); - uint32_t completed_jobs = 0, key_len = 0; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - vec->digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (vec->key_len <= block_size) { - memcpy(key, vec->key, vec->key_len); - key_len = (int) vec->key_len; - } else { - IMB_SHA1(mb_mgr, vec->key, vec->key_len, key); - key_len = digest_size; - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); - - for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; - job->enc_keys = NULL; - job->dec_keys = NULL; - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->dst = NULL; - job->key_len_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->digest_len; - job->iv = NULL; - job->iv_len_in_bytes = 0; - job->src = vec->data; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = 0; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; - job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_HMAC_SHA_1; - - job->user_data = auths[i]; - - } - - completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, - IMB_AUTH_HMAC_SHA_1); - if (completed_jobs != num_jobs) { - int err = imb_get_errno(mb_mgr); - - if (err != 0) { - printf("submit_burst error %d : '%s'\n", err, - imb_get_strerror(err)); - goto end; - } else { - printf("submit_burst error: not enough " - "jobs returned!\n"); - goto end; - } - } - - for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; - - if (job->status != IMB_STATUS_COMPLETED) { - printf("job %u status not complete!\n", i+1); - goto end; - } - - if (!hmac_sha1_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - jobs_rx++; - } - - if (jobs_rx != num_jobs) { - printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); - goto end; - } - ret = 0; - - end: - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static void -test_hmac_sha1_std_vectors(struct IMB_MGR *mb_mgr, - const uint32_t num_jobs, - struct test_suite_context *ts) -{ - const int vectors_cnt = DIM(hmac_sha1_vectors); - int vect; - - printf("HMAC-SHA1 standard test vectors (N jobs = %u):\n", num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("[%d/%d] RFC2202 Test Case %s key_len:%d data_len:%d " - "digest_len:%d\n", - vect, vectors_cnt, - hmac_sha1_vectors[idx].test_case, - (int) hmac_sha1_vectors[idx].key_len, - (int) hmac_sha1_vectors[idx].data_len, - (int) hmac_sha1_vectors[idx].digest_len); -#else - printf("."); -#endif - - if (test_hmac_sha1(mb_mgr, &hmac_sha1_vectors[idx], num_jobs)) { - printf("error #%d\n", vect); - test_suite_update(ts, 0, 1); - } else { - test_suite_update(ts, 1, 0); - } - if (test_hmac_sha1_burst(mb_mgr, &hmac_sha1_vectors[idx], - num_jobs)) { - printf("error #%d - burst API\n", vect); - test_suite_update(ts, 0, 1); - } else { - test_suite_update(ts, 1, 0); - } - if (test_hmac_sha1_hash_burst(mb_mgr, &hmac_sha1_vectors[idx], - num_jobs)) { - printf("error #%d - hash-only burst API\n", vect); - test_suite_update(ts, 0, 1); - } else { - test_suite_update(ts, 1, 0); - } - } - printf("\n"); -} - -int -hmac_sha1_test(struct IMB_MGR *mb_mgr) -{ - struct test_suite_context ts; - int errors = 0; - uint32_t num_jobs; - - test_suite_start(&ts, "HMAC-SHA1"); - for (num_jobs = 1; num_jobs <= max_burst_jobs; num_jobs++) - test_hmac_sha1_std_vectors(mb_mgr, num_jobs, &ts); - errors = test_suite_end(&ts); - - return errors; -} diff --git a/test/hmac_sha256_sha512_test.c b/test/hmac_sha256_sha512_test.c deleted file mode 100644 index c8ba3c3a55a57c3ae40f4d2c9ab9b96e3df90a05..0000000000000000000000000000000000000000 --- a/test/hmac_sha256_sha512_test.c +++ /dev/null @@ -1,1591 +0,0 @@ -/***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include -#include -#include - -#include -#include "gcm_ctr_vectors_test.h" -#include "utils.h" - -#define max_burst_jobs 32 - -int hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr); - -/* - * Test vectors from https://tools.ietf.org/html/rfc4231 - */ - -/* - * 4.2. Test Case 1 - * - * Key = 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b - * 0b0b0b0b (20 bytes) - * Data = 4869205468657265 ("Hi There") - * - * HMAC-SHA-224 = 896fb1128abbdf196832107cd49df33f - * 47b4b1169912ba4f53684b22 - * HMAC-SHA-256 = b0344c61d8db38535ca8afceaf0bf12b - * 881dc200c9833da726e9376c2e32cff7 - * HMAC-SHA-384 = afd03944d84895626b0825f4ab46907f - * 15f9dadbe4101ec682aa034c7cebc59c - * faea9ea9076ede7f4af152e8b2fa9cb6 - * HMAC-SHA-512 = 87aa7cdea5ef619d4ff0b4241a1d6cb0 - * 2379f4e2ce4ec2787ad0b30545e17cde - * daa833b7d6b8a702038b274eaea3f4e4 - * be9d914eeb61f1702e696c203a126854 - */ -static const uint8_t key_1[] = { - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, - 0x0b, 0x0b, 0x0b, 0x0b -}; -static const uint8_t data_1[] = { - 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65 -}; -static const uint8_t hmac_sha_224_1[] = { - 0x89, 0x6f, 0xb1, 0x12, 0x8a, 0xbb, 0xdf, 0x19, - 0x68, 0x32, 0x10, 0x7c, 0xd4, 0x9d, 0xf3, 0x3f, - 0x47, 0xb4, 0xb1, 0x16, 0x99, 0x12, 0xba, 0x4f, - 0x53, 0x68, 0x4b, 0x22 -}; -static const uint8_t hmac_sha_256_1[] = { - 0xb0, 0x34, 0x4c, 0x61, 0xd8, 0xdb, 0x38, 0x53, - 0x5c, 0xa8, 0xaf, 0xce, 0xaf, 0x0b, 0xf1, 0x2b, - 0x88, 0x1d, 0xc2, 0x00, 0xc9, 0x83, 0x3d, 0xa7, - 0x26, 0xe9, 0x37, 0x6c, 0x2e, 0x32, 0xcf, 0xf7 -}; -static const uint8_t hmac_sha_384_1[] = { - 0xaf, 0xd0, 0x39, 0x44, 0xd8, 0x48, 0x95, 0x62, - 0x6b, 0x08, 0x25, 0xf4, 0xab, 0x46, 0x90, 0x7f, - 0x15, 0xf9, 0xda, 0xdb, 0xe4, 0x10, 0x1e, 0xc6, - 0x82, 0xaa, 0x03, 0x4c, 0x7c, 0xeb, 0xc5, 0x9c, - 0xfa, 0xea, 0x9e, 0xa9, 0x07, 0x6e, 0xde, 0x7f, - 0x4a, 0xf1, 0x52, 0xe8, 0xb2, 0xfa, 0x9c, 0xb6 -}; -static const uint8_t hmac_sha_512_1[] = { - 0x87, 0xaa, 0x7c, 0xde, 0xa5, 0xef, 0x61, 0x9d, - 0x4f, 0xf0, 0xb4, 0x24, 0x1a, 0x1d, 0x6c, 0xb0, - 0x23, 0x79, 0xf4, 0xe2, 0xce, 0x4e, 0xc2, 0x78, - 0x7a, 0xd0, 0xb3, 0x05, 0x45, 0xe1, 0x7c, 0xde, - 0xda, 0xa8, 0x33, 0xb7, 0xd6, 0xb8, 0xa7, 0x02, - 0x03, 0x8b, 0x27, 0x4e, 0xae, 0xa3, 0xf4, 0xe4, - 0xbe, 0x9d, 0x91, 0x4e, 0xeb, 0x61, 0xf1, 0x70, - 0x2e, 0x69, 0x6c, 0x20, 0x3a, 0x12, 0x68, 0x54 -}; - -/* - * 4.3. Test Case 2 - * - * Test with a key shorter than the length of the HMAC output. - * - * Key = 4a656665 ("Jefe") - * Data = 7768617420646f2079612077616e7420 ("what do ya want ") - * 666f72206e6f7468696e673f ("for nothing?") - * - * HMAC-SHA-224 = a30e01098bc6dbbf45690f3a7e9e6d0f - * 8bbea2a39e6148008fd05e44 - * HMAC-SHA-256 = 5bdcc146bf60754e6a042426089575c7 - * 5a003f089d2739839dec58b964ec3843 - * HMAC-SHA-384 = af45d2e376484031617f78d2b58a6b1b - * 9c7ef464f5a01b47e42ec3736322445e - * 8e2240ca5e69e2c78b3239ecfab21649 - * HMAC-SHA-512 = 164b7a7bfcf819e2e395fbe73b56e0a3 - * 87bd64222e831fd610270cd7ea250554 - * 9758bf75c05a994a6d034f65f8f0e6fd - * caeab1a34d4a6b4b636e070a38bce737 - */ -static const uint8_t key_2[] = { - 0x4a, 0x65, 0x66, 0x65 -}; -static const uint8_t data_2[] = { - 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, - 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, - 0x69, 0x6e, 0x67, 0x3f -}; -static const uint8_t hmac_sha_224_2[] = { - 0xa3, 0x0e, 0x01, 0x09, 0x8b, 0xc6, 0xdb, 0xbf, - 0x45, 0x69, 0x0f, 0x3a, 0x7e, 0x9e, 0x6d, 0x0f, - 0x8b, 0xbe, 0xa2, 0xa3, 0x9e, 0x61, 0x48, 0x00, - 0x8f, 0xd0, 0x5e, 0x44 -}; -static const uint8_t hmac_sha_256_2[] = { - 0x5b, 0xdc, 0xc1, 0x46, 0xbf, 0x60, 0x75, 0x4e, - 0x6a, 0x04, 0x24, 0x26, 0x08, 0x95, 0x75, 0xc7, - 0x5a, 0x00, 0x3f, 0x08, 0x9d, 0x27, 0x39, 0x83, - 0x9d, 0xec, 0x58, 0xb9, 0x64, 0xec, 0x38, 0x43 -}; -static const uint8_t hmac_sha_384_2[] = { - 0xaf, 0x45, 0xd2, 0xe3, 0x76, 0x48, 0x40, 0x31, - 0x61, 0x7f, 0x78, 0xd2, 0xb5, 0x8a, 0x6b, 0x1b, - 0x9c, 0x7e, 0xf4, 0x64, 0xf5, 0xa0, 0x1b, 0x47, - 0xe4, 0x2e, 0xc3, 0x73, 0x63, 0x22, 0x44, 0x5e, - 0x8e, 0x22, 0x40, 0xca, 0x5e, 0x69, 0xe2, 0xc7, - 0x8b, 0x32, 0x39, 0xec, 0xfa, 0xb2, 0x16, 0x49 -}; -static const uint8_t hmac_sha_512_2[] = { - 0x16, 0x4b, 0x7a, 0x7b, 0xfc, 0xf8, 0x19, 0xe2, - 0xe3, 0x95, 0xfb, 0xe7, 0x3b, 0x56, 0xe0, 0xa3, - 0x87, 0xbd, 0x64, 0x22, 0x2e, 0x83, 0x1f, 0xd6, - 0x10, 0x27, 0x0c, 0xd7, 0xea, 0x25, 0x05, 0x54, - 0x97, 0x58, 0xbf, 0x75, 0xc0, 0x5a, 0x99, 0x4a, - 0x6d, 0x03, 0x4f, 0x65, 0xf8, 0xf0, 0xe6, 0xfd, - 0xca, 0xea, 0xb1, 0xa3, 0x4d, 0x4a, 0x6b, 0x4b, - 0x63, 0x6e, 0x07, 0x0a, 0x38, 0xbc, 0xe7, 0x37 -}; - -/* - * 4.4. Test Case 3 - * - * Test with a combined length of key and data that is larger than 64 - * bytes (= block-size of SHA-224 and SHA-256). - * - * Key aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaa (20 bytes) - * Data = dddddddddddddddddddddddddddddddd - * dddddddddddddddddddddddddddddddd - * dddddddddddddddddddddddddddddddd - * dddd (50 bytes) - * - * HMAC-SHA-224 = 7fb3cb3588c6c1f6ffa9694d7d6ad264 - * 9365b0c1f65d69d1ec8333ea - * HMAC-SHA-256 = 773ea91e36800e46854db8ebd09181a7 - * 2959098b3ef8c122d9635514ced565fe - * HMAC-SHA-384 = 88062608d3e6ad8a0aa2ace014c8a86f - * 0aa635d947ac9febe83ef4e55966144b - * 2a5ab39dc13814b94e3ab6e101a34f27 - * HMAC-SHA-512 = fa73b0089d56a284efb0f0756c890be9 - * b1b5dbdd8ee81a3655f83e33b2279d39 - * bf3e848279a722c806b485a47e67c807 - * b946a337bee8942674278859e13292fb - */ -static const uint8_t key_3[] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa -}; -static const uint8_t data_3[] = { - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, - 0xdd, 0xdd -}; -static const uint8_t hmac_sha_224_3[] = { - 0x7f, 0xb3, 0xcb, 0x35, 0x88, 0xc6, 0xc1, 0xf6, - 0xff, 0xa9, 0x69, 0x4d, 0x7d, 0x6a, 0xd2, 0x64, - 0x93, 0x65, 0xb0, 0xc1, 0xf6, 0x5d, 0x69, 0xd1, - 0xec, 0x83, 0x33, 0xea -}; -static const uint8_t hmac_sha_256_3[] = { - 0x77, 0x3e, 0xa9, 0x1e, 0x36, 0x80, 0x0e, 0x46, - 0x85, 0x4d, 0xb8, 0xeb, 0xd0, 0x91, 0x81, 0xa7, - 0x29, 0x59, 0x09, 0x8b, 0x3e, 0xf8, 0xc1, 0x22, - 0xd9, 0x63, 0x55, 0x14, 0xce, 0xd5, 0x65, 0xfe -}; -static const uint8_t hmac_sha_384_3[] = { - 0x88, 0x06, 0x26, 0x08, 0xd3, 0xe6, 0xad, 0x8a, - 0x0a, 0xa2, 0xac, 0xe0, 0x14, 0xc8, 0xa8, 0x6f, - 0x0a, 0xa6, 0x35, 0xd9, 0x47, 0xac, 0x9f, 0xeb, - 0xe8, 0x3e, 0xf4, 0xe5, 0x59, 0x66, 0x14, 0x4b, - 0x2a, 0x5a, 0xb3, 0x9d, 0xc1, 0x38, 0x14, 0xb9, - 0x4e, 0x3a, 0xb6, 0xe1, 0x01, 0xa3, 0x4f, 0x27 -}; -static const uint8_t hmac_sha_512_3[] = { - 0xfa, 0x73, 0xb0, 0x08, 0x9d, 0x56, 0xa2, 0x84, - 0xef, 0xb0, 0xf0, 0x75, 0x6c, 0x89, 0x0b, 0xe9, - 0xb1, 0xb5, 0xdb, 0xdd, 0x8e, 0xe8, 0x1a, 0x36, - 0x55, 0xf8, 0x3e, 0x33, 0xb2, 0x27, 0x9d, 0x39, - 0xbf, 0x3e, 0x84, 0x82, 0x79, 0xa7, 0x22, 0xc8, - 0x06, 0xb4, 0x85, 0xa4, 0x7e, 0x67, 0xc8, 0x07, - 0xb9, 0x46, 0xa3, 0x37, 0xbe, 0xe8, 0x94, 0x26, - 0x74, 0x27, 0x88, 0x59, 0xe1, 0x32, 0x92, 0xfb -}; - -/* - * 4.5. Test Case 4 - * - * Test with a combined length of key and data that is larger than 64 - * bytes (= block-size of SHA-224 and SHA-256). - * - * Key = 0102030405060708090a0b0c0d0e0f10 - * 111213141516171819 (25 bytes) - * Data = cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd - * cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd - * cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd - * cdcd (50 bytes) - * - * HMAC-SHA-224 = 6c11506874013cac6a2abc1bb382627c - * ec6a90d86efc012de7afec5a - * HMAC-SHA-256 = 82558a389a443c0ea4cc819899f2083a - * 85f0faa3e578f8077a2e3ff46729665b - * HMAC-SHA-384 = 3e8a69b7783c25851933ab6290af6ca7 - * 7a9981480850009cc5577c6e1f573b4e - * 6801dd23c4a7d679ccf8a386c674cffb - * HMAC-SHA-512 = b0ba465637458c6990e5a8c5f61d4af7 - * e576d97ff94b872de76f8050361ee3db - * a91ca5c11aa25eb4d679275cc5788063 - * a5f19741120c4f2de2adebeb10a298dd - */ -static const uint8_t key_4[] = { - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, - 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, - 0x19 -}; -static const uint8_t data_4[] = { - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd -}; -static const uint8_t hmac_sha_224_4[] = { - 0x6c, 0x11, 0x50, 0x68, 0x74, 0x01, 0x3c, 0xac, - 0x6a, 0x2a, 0xbc, 0x1b, 0xb3, 0x82, 0x62, 0x7c, - 0xec, 0x6a, 0x90, 0xd8, 0x6e, 0xfc, 0x01, 0x2d, - 0xe7, 0xaf, 0xec, 0x5a -}; -static const uint8_t hmac_sha_256_4[] = { - 0x82, 0x55, 0x8a, 0x38, 0x9a, 0x44, 0x3c, 0x0e, - 0xa4, 0xcc, 0x81, 0x98, 0x99, 0xf2, 0x08, 0x3a, - 0x85, 0xf0, 0xfa, 0xa3, 0xe5, 0x78, 0xf8, 0x07, - 0x7a, 0x2e, 0x3f, 0xf4, 0x67, 0x29, 0x66, 0x5b -}; -static const uint8_t hmac_sha_384_4[] = { - 0x3e, 0x8a, 0x69, 0xb7, 0x78, 0x3c, 0x25, 0x85, - 0x19, 0x33, 0xab, 0x62, 0x90, 0xaf, 0x6c, 0xa7, - 0x7a, 0x99, 0x81, 0x48, 0x08, 0x50, 0x00, 0x9c, - 0xc5, 0x57, 0x7c, 0x6e, 0x1f, 0x57, 0x3b, 0x4e, - 0x68, 0x01, 0xdd, 0x23, 0xc4, 0xa7, 0xd6, 0x79, - 0xcc, 0xf8, 0xa3, 0x86, 0xc6, 0x74, 0xcf, 0xfb -}; -static const uint8_t hmac_sha_512_4[] = { - 0xb0, 0xba, 0x46, 0x56, 0x37, 0x45, 0x8c, 0x69, - 0x90, 0xe5, 0xa8, 0xc5, 0xf6, 0x1d, 0x4a, 0xf7, - 0xe5, 0x76, 0xd9, 0x7f, 0xf9, 0x4b, 0x87, 0x2d, - 0xe7, 0x6f, 0x80, 0x50, 0x36, 0x1e, 0xe3, 0xdb, - 0xa9, 0x1c, 0xa5, 0xc1, 0x1a, 0xa2, 0x5e, 0xb4, - 0xd6, 0x79, 0x27, 0x5c, 0xc5, 0x78, 0x80, 0x63, - 0xa5, 0xf1, 0x97, 0x41, 0x12, 0x0c, 0x4f, 0x2d, - 0xe2, 0xad, 0xeb, 0xeb, 0x10, 0xa2, 0x98, 0xdd -}; - -/* - * - * 4.6. Test Case 5 - * - * Test with a truncation of output to 128 bits. - * - * Key = 0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c - * 0c0c0c0c (20 bytes) - * Data = 546573742057697468205472756e6361 ("Test With Trunca") - * 74696f6e ("tion") - * - * HMAC-SHA-224 = 0e2aea68a90c8d37c988bcdb9fca6fa8 - * HMAC-SHA-256 = a3b6167473100ee06e0c796c2955552b - * HMAC-SHA-384 = 3abf34c3503b2a23a46efc619baef897 - * HMAC-SHA-512 = 415fad6271580a531d4179bc891d87a6 - */ -/* static const uint8_t key_5[] = { */ -/* 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, */ -/* 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, */ -/* 0x0c, 0x0c, 0x0c, 0x0c */ -/* }; */ -/* static const uint8_t data_5[] = { */ -/* 0x54, 0x65, 0x73, 0x74, 0x20, 0x57, 0x69, 0x74, */ -/* 0x68, 0x20, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, */ -/* 0x74, 0x69, 0x6f, 0x6e */ -/* }; */ -/* static const uint8_t hmac_sha_224_5[] = { */ -/* 0x0e, 0x2a, 0xea, 0x68, 0xa9, 0x0c, 0x8d, 0x37, */ -/* 0xc9, 0x88, 0xbc, 0xdb, 0x9f, 0xca, 0x6f, 0xa8 */ -/* }; */ -/* static const uint8_t hmac_sha_256_5[] = { */ -/* 0xa3, 0xb6, 0x16, 0x74, 0x73, 0x10, 0x0e, 0xe0, */ -/* 0x6e, 0x0c, 0x79, 0x6c, 0x29, 0x55, 0x55, 0x2b */ -/* }; */ -/* static const uint8_t hmac_sha_384_5[] = { */ -/* 0x3a, 0xbf, 0x34, 0xc3, 0x50, 0x3b, 0x2a, 0x23, */ -/* 0xa4, 0x6e, 0xfc, 0x61, 0x9b, 0xae, 0xf8, 0x97 */ -/* }; */ -/* static const uint8_t hmac_sha_512_5[] = { */ -/* 0x41, 0x5f, 0xad, 0x62, 0x71, 0x58, 0x0a, 0x53, */ -/* 0x1d, 0x41, 0x79, 0xbc, 0x89, 0x1d, 0x87, 0xa6 */ -/* }; */ - -/* - * 4.7. Test Case 6 - * - * Test with a key larger than 128 bytes (= block-size of SHA-384 and - * SHA-512). - * - * Key = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaa (131 bytes) - * Data = 54657374205573696e67204c61726765 ("Test Using Large") - * 72205468616e20426c6f636b2d53697a ("r Than Block-Siz") - * 65204b6579202d2048617368204b6579 ("e Key - Hash Key") - * 204669727374 (" First") - * - * HMAC-SHA-224 = 95e9a0db962095adaebe9b2d6f0dbce2 - * d499f112f2d2b7273fa6870e - * HMAC-SHA-256 = 60e431591ee0b67f0d8a26aacbf5b77f - * 8e0bc6213728c5140546040f0ee37f54 - * HMAC-SHA-384 = 4ece084485813e9088d2c63a041bc5b4 - * 4f9ef1012a2b588f3cd11f05033ac4c6 - * 0c2ef6ab4030fe8296248df163f44952 - * HMAC-SHA-512 = 80b24263c7c1a3ebb71493c1dd7be8b4 - * 9b46d1f41b4aeec1121b013783f8f352 - * 6b56d037e05f2598bd0fd2215d6a1e52 - * 95e64f73f63f0aec8b915a985d786598 - */ -static const uint8_t key_6[] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa -}; -static const uint8_t data_6[] = { - 0x54, 0x65, 0x73, 0x74, 0x20, 0x55, 0x73, 0x69, - 0x6e, 0x67, 0x20, 0x4c, 0x61, 0x72, 0x67, 0x65, - 0x72, 0x20, 0x54, 0x68, 0x61, 0x6e, 0x20, 0x42, - 0x6c, 0x6f, 0x63, 0x6b, 0x2d, 0x53, 0x69, 0x7a, - 0x65, 0x20, 0x4b, 0x65, 0x79, 0x20, 0x2d, 0x20, - 0x48, 0x61, 0x73, 0x68, 0x20, 0x4b, 0x65, 0x79, - 0x20, 0x46, 0x69, 0x72, 0x73, 0x74 -}; -static const uint8_t hmac_sha_224_6[] = { - 0x95, 0xe9, 0xa0, 0xdb, 0x96, 0x20, 0x95, 0xad, - 0xae, 0xbe, 0x9b, 0x2d, 0x6f, 0x0d, 0xbc, 0xe2, - 0xd4, 0x99, 0xf1, 0x12, 0xf2, 0xd2, 0xb7, 0x27, - 0x3f, 0xa6, 0x87, 0x0e -}; -static const uint8_t hmac_sha_256_6[] = { - 0x60, 0xe4, 0x31, 0x59, 0x1e, 0xe0, 0xb6, 0x7f, - 0x0d, 0x8a, 0x26, 0xaa, 0xcb, 0xf5, 0xb7, 0x7f, - 0x8e, 0x0b, 0xc6, 0x21, 0x37, 0x28, 0xc5, 0x14, - 0x05, 0x46, 0x04, 0x0f, 0x0e, 0xe3, 0x7f, 0x54 -}; -static const uint8_t hmac_sha_384_6[] = { - 0x4e, 0xce, 0x08, 0x44, 0x85, 0x81, 0x3e, 0x90, - 0x88, 0xd2, 0xc6, 0x3a, 0x04, 0x1b, 0xc5, 0xb4, - 0x4f, 0x9e, 0xf1, 0x01, 0x2a, 0x2b, 0x58, 0x8f, - 0x3c, 0xd1, 0x1f, 0x05, 0x03, 0x3a, 0xc4, 0xc6, - 0x0c, 0x2e, 0xf6, 0xab, 0x40, 0x30, 0xfe, 0x82, - 0x96, 0x24, 0x8d, 0xf1, 0x63, 0xf4, 0x49, 0x52 -}; -static const uint8_t hmac_sha_512_6[] = { - 0x80, 0xb2, 0x42, 0x63, 0xc7, 0xc1, 0xa3, 0xeb, - 0xb7, 0x14, 0x93, 0xc1, 0xdd, 0x7b, 0xe8, 0xb4, - 0x9b, 0x46, 0xd1, 0xf4, 0x1b, 0x4a, 0xee, 0xc1, - 0x12, 0x1b, 0x01, 0x37, 0x83, 0xf8, 0xf3, 0x52, - 0x6b, 0x56, 0xd0, 0x37, 0xe0, 0x5f, 0x25, 0x98, - 0xbd, 0x0f, 0xd2, 0x21, 0x5d, 0x6a, 0x1e, 0x52, - 0x95, 0xe6, 0x4f, 0x73, 0xf6, 0x3f, 0x0a, 0xec, - 0x8b, 0x91, 0x5a, 0x98, 0x5d, 0x78, 0x65, 0x98 -}; - -/* - * 4.8. Test Case 7 - * - * Test with a key and data that is larger than 128 bytes (= block-size - * of SHA-384 and SHA-512). - * - * Key = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - * aaaaaa (131 bytes) - * Data = 54686973206973206120746573742075 ("This is a test u") - * 73696e672061206c6172676572207468 ("sing a larger th") - * 616e20626c6f636b2d73697a65206b65 ("an block-size ke") - * 7920616e642061206c61726765722074 ("y and a larger t") - * 68616e20626c6f636b2d73697a652064 ("han block-size d") - * 6174612e20546865206b6579206e6565 ("ata. The key nee") - * 647320746f2062652068617368656420 ("ds to be hashed ") - * 6265666f7265206265696e6720757365 ("before being use") - * 642062792074686520484d414320616c ("d by the HMAC al") - * 676f726974686d2e ("gorithm.") - * - * HMAC-SHA-224 = 3a854166ac5d9f023f54d517d0b39dbd - * 946770db9c2b95c9f6f565d1 - * HMAC-SHA-256 = 9b09ffa71b942fcb27635fbcd5b0e944 - * bfdc63644f0713938a7f51535c3a35e2 - * HMAC-SHA-384 = 6617178e941f020d351e2f254e8fd32c - * 602420feb0b8fb9adccebb82461e99c5 - * a678cc31e799176d3860e6110c46523e - * HMAC-SHA-512 = e37b6a775dc87dbaa4dfa9f96e5e3ffd - * debd71f8867289865df5a32d20cdc944 - * b6022cac3c4982b10d5eeb55c3e4de15 - * 134676fb6de0446065c97440fa8c6a58 - */ -static const uint8_t key_7[] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xaa -}; -static const uint8_t data_7[] = { - 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, - 0x61, 0x20, 0x74, 0x65, 0x73, 0x74, 0x20, 0x75, - 0x73, 0x69, 0x6e, 0x67, 0x20, 0x61, 0x20, 0x6c, - 0x61, 0x72, 0x67, 0x65, 0x72, 0x20, 0x74, 0x68, - 0x61, 0x6e, 0x20, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x2d, 0x73, 0x69, 0x7a, 0x65, 0x20, 0x6b, 0x65, - 0x79, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x61, 0x20, - 0x6c, 0x61, 0x72, 0x67, 0x65, 0x72, 0x20, 0x74, - 0x68, 0x61, 0x6e, 0x20, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x2d, 0x73, 0x69, 0x7a, 0x65, 0x20, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x20, 0x54, 0x68, 0x65, - 0x20, 0x6b, 0x65, 0x79, 0x20, 0x6e, 0x65, 0x65, - 0x64, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x65, - 0x20, 0x68, 0x61, 0x73, 0x68, 0x65, 0x64, 0x20, - 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x20, 0x62, - 0x65, 0x69, 0x6e, 0x67, 0x20, 0x75, 0x73, 0x65, - 0x64, 0x20, 0x62, 0x79, 0x20, 0x74, 0x68, 0x65, - 0x20, 0x48, 0x4d, 0x41, 0x43, 0x20, 0x61, 0x6c, - 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x2e -}; -static const uint8_t hmac_sha_224_7[] = { - 0x3a, 0x85, 0x41, 0x66, 0xac, 0x5d, 0x9f, 0x02, - 0x3f, 0x54, 0xd5, 0x17, 0xd0, 0xb3, 0x9d, 0xbd, - 0x94, 0x67, 0x70, 0xdb, 0x9c, 0x2b, 0x95, 0xc9, - 0xf6, 0xf5, 0x65, 0xd1 -}; -static const uint8_t hmac_sha_256_7[] = { - 0x9b, 0x09, 0xff, 0xa7, 0x1b, 0x94, 0x2f, 0xcb, - 0x27, 0x63, 0x5f, 0xbc, 0xd5, 0xb0, 0xe9, 0x44, - 0xbf, 0xdc, 0x63, 0x64, 0x4f, 0x07, 0x13, 0x93, - 0x8a, 0x7f, 0x51, 0x53, 0x5c, 0x3a, 0x35, 0xe2 -}; -static const uint8_t hmac_sha_384_7[] = { - 0x66, 0x17, 0x17, 0x8e, 0x94, 0x1f, 0x02, 0x0d, - 0x35, 0x1e, 0x2f, 0x25, 0x4e, 0x8f, 0xd3, 0x2c, - 0x60, 0x24, 0x20, 0xfe, 0xb0, 0xb8, 0xfb, 0x9a, - 0xdc, 0xce, 0xbb, 0x82, 0x46, 0x1e, 0x99, 0xc5, - 0xa6, 0x78, 0xcc, 0x31, 0xe7, 0x99, 0x17, 0x6d, - 0x38, 0x60, 0xe6, 0x11, 0x0c, 0x46, 0x52, 0x3e -}; -static const uint8_t hmac_sha_512_7[] = { - 0xe3, 0x7b, 0x6a, 0x77, 0x5d, 0xc8, 0x7d, 0xba, - 0xa4, 0xdf, 0xa9, 0xf9, 0x6e, 0x5e, 0x3f, 0xfd, - 0xde, 0xbd, 0x71, 0xf8, 0x86, 0x72, 0x89, 0x86, - 0x5d, 0xf5, 0xa3, 0x2d, 0x20, 0xcd, 0xc9, 0x44, - 0xb6, 0x02, 0x2c, 0xac, 0x3c, 0x49, 0x82, 0xb1, - 0x0d, 0x5e, 0xeb, 0x55, 0xc3, 0xe4, 0xde, 0x15, - 0x13, 0x46, 0x76, 0xfb, 0x6d, 0xe0, 0x44, 0x60, - 0x65, 0xc9, 0x74, 0x40, 0xfa, 0x8c, 0x6a, 0x58 -}; - -/* - * Test Case 8 - * - * Test vector from https://csrc.nist.gov/csrc/media/projects/ - * cryptographic-standards-and-guidelines/documents/examples/hmac_sha224.pdf - */ -static const uint8_t key_8[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f -}; -static const uint8_t data_8[] = { - 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, - 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, - 0x65, 0x6e -}; -static const uint8_t hmac_sha_224_8[] = { - 0xc7, 0x40, 0x5e, 0x3a, 0xe0, 0x58, 0xe8, 0xcd, - 0x30, 0xb0, 0x8b, 0x41, 0x40, 0x24, 0x85, 0x81, - 0xed, 0x17, 0x4c, 0xb3, 0x4e, 0x12, 0x24, 0xbc, - 0xc1, 0xef, 0xc8, 0x1b -}; - -/* - * Test Case 9 - * - * Test vector from https://csrc.nist.gov/csrc/media/projects/ - * cryptographic-standards-and-guidelines/documents/examples/hmac_sha256.pdf - */ -static const uint8_t key_9[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f -}; -static const uint8_t data_9[] = { - 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, - 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, - 0x65, 0x6e -}; -static const uint8_t hmac_sha_256_9[] = { - 0x8b, 0xb9, 0xa1, 0xdb, 0x98, 0x06, 0xf2, 0x0d, - 0xf7, 0xf7, 0x7b, 0x82, 0x13, 0x8c, 0x79, 0x14, - 0xd1, 0x74, 0xd5, 0x9e, 0x13, 0xdc, 0x4d, 0x01, - 0x69, 0xc9, 0x05, 0x7b, 0x13, 0x3e, 0x1d, 0x62, -}; - -/* - * Test Case 10 - * - * Test vector from https://csrc.nist.gov/csrc/media/projects/ - * cryptographic-standards-and-guidelines/documents/examples/hmac_sha384.pdf - */ -static const uint8_t key_10[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f -}; -static const uint8_t data_10[] = { - 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, - 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, - 0x65, 0x6e -}; -static const uint8_t hmac_sha_384_10[] = { - 0x63, 0xc5, 0xda, 0xa5, 0xe6, 0x51, 0x84, 0x7c, - 0xa8, 0x97, 0xc9, 0x58, 0x14, 0xab, 0x83, 0x0b, - 0xed, 0xed, 0xc7, 0xd2, 0x5e, 0x83, 0xee, 0xf9 -}; - -/* - * Test Case 11 - * - * Test vector from https://csrc.nist.gov/csrc/media/projects/ - * cryptographic-standards-and-guidelines/documents/examples/hmac_sha512.pdf - */ -static const uint8_t key_11[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f -}; -static const uint8_t data_11[] = { - 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, - 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, - 0x65, 0x6e -}; -static const uint8_t hmac_sha_512_11[] = { - 0xfc, 0x25, 0xe2, 0x40, 0x65, 0x8c, 0xa7, 0x85, - 0xb7, 0xa8, 0x11, 0xa8, 0xd3, 0xf7, 0xb4, 0xca, - 0x48, 0xcf, 0xa2, 0x6a, 0x8a, 0x36, 0x6b, 0xf2, - 0xcd, 0x1f, 0x83, 0x6b, 0x05, 0xfc, 0xb0, 0x24 -}; - -#define HMAC_SHA256_SHA512_TEST_VEC(num) \ - { num, \ - key_##num, sizeof(key_##num), \ - data_##num, sizeof(data_##num), \ - hmac_sha_224_##num, sizeof(hmac_sha_224_##num), \ - hmac_sha_256_##num, sizeof(hmac_sha_256_##num), \ - hmac_sha_384_##num, sizeof(hmac_sha_384_##num), \ - hmac_sha_512_##num, sizeof(hmac_sha_512_##num) } - -#define HMAC_SHA224_TEST_VEC(num) \ - { num, \ - key_##num, sizeof(key_##num), \ - data_##num, sizeof(data_##num), \ - hmac_sha_224_##num, sizeof(hmac_sha_224_##num), \ - NULL, 0, \ - NULL, 0, \ - NULL, 0 } - -#define HMAC_SHA256_TEST_VEC(num) \ - { num, \ - key_##num, sizeof(key_##num), \ - data_##num, sizeof(data_##num), \ - NULL, 0, \ - hmac_sha_256_##num, sizeof(hmac_sha_256_##num), \ - NULL, 0, \ - NULL, 0 } - -#define HMAC_SHA384_TEST_VEC(num) \ - { num, \ - key_##num, sizeof(key_##num), \ - data_##num, sizeof(data_##num), \ - NULL, 0, \ - NULL, 0, \ - hmac_sha_384_##num, sizeof(hmac_sha_384_##num), \ - NULL, 0 } - -#define HMAC_SHA512_TEST_VEC(num) \ - { num, \ - key_##num, sizeof(key_##num), \ - data_##num, sizeof(data_##num), \ - NULL, 0, \ - NULL, 0, \ - NULL, 0, \ - hmac_sha_512_##num, sizeof(hmac_sha_512_##num) } - -static const struct hmac_rfc4231_vector { - int test_case_num; - const uint8_t *key; - size_t key_len; - const uint8_t *data; - size_t data_len; - const uint8_t *hmac_sha224; - size_t hmac_sha224_len; - const uint8_t *hmac_sha256; - size_t hmac_sha256_len; - const uint8_t *hmac_sha384; - size_t hmac_sha384_len; - const uint8_t *hmac_sha512; - size_t hmac_sha512_len; -} hmac_sha256_sha512_vectors[] = { - HMAC_SHA256_SHA512_TEST_VEC(1), - HMAC_SHA256_SHA512_TEST_VEC(2), - HMAC_SHA256_SHA512_TEST_VEC(3), - HMAC_SHA256_SHA512_TEST_VEC(4), - /* HMAC_SHA256_SHA512_TEST_VEC(5), */ - HMAC_SHA256_SHA512_TEST_VEC(6), - HMAC_SHA256_SHA512_TEST_VEC(7), - HMAC_SHA224_TEST_VEC(8), - HMAC_SHA256_TEST_VEC(9), - HMAC_SHA384_TEST_VEC(10), - HMAC_SHA512_TEST_VEC(11), -}; - -static int -hmac_shax_job_ok(const struct hmac_rfc4231_vector *vec, - const struct IMB_JOB *job, - const int sha_type, - const uint8_t *auth, - const uint8_t *padding, - const size_t sizeof_padding) -{ - const uint8_t *p_digest = NULL; - size_t digest_len = 0; - - switch (sha_type) { - case 224: - p_digest = vec->hmac_sha224; - digest_len = vec->hmac_sha224_len; - break; - case 256: - p_digest = vec->hmac_sha256; - digest_len = vec->hmac_sha256_len; - break; - case 384: - p_digest = vec->hmac_sha384; - digest_len = vec->hmac_sha384_len; - break; - case 512: - p_digest = vec->hmac_sha512; - digest_len = vec->hmac_sha512_len; - break; - default: - printf("line:%d wrong SHA type 'SHA-%d' ", __LINE__, sha_type); - return 0; - break; - } - - if (job->status != IMB_STATUS_COMPLETED) { - printf("line:%d job error status:%d ", __LINE__, job->status); - return 0; - } - - /* hash checks */ - if (memcmp(padding, &auth[sizeof_padding + digest_len], - sizeof_padding)) { - printf("hash overwrite tail\n"); - hexdump(stderr, "Target", - &auth[sizeof_padding + digest_len], - sizeof_padding); - return 0; - } - - if (memcmp(padding, &auth[0], sizeof_padding)) { - printf("hash overwrite head\n"); - hexdump(stderr, "Target", &auth[0], sizeof_padding); - return 0; - } - - if (memcmp(p_digest, &auth[sizeof_padding], digest_len)) { - printf("hash mismatched\n"); - hexdump(stderr, "Received", &auth[sizeof_padding], digest_len); - hexdump(stderr, "Expected", p_digest, digest_len); - return 0; - } - return 1; -} - -static int -test_hmac_shax(struct IMB_MGR *mb_mgr, - const struct hmac_rfc4231_vector *vec, - const uint32_t num_jobs, - const int sha_type) -{ - struct IMB_JOB *job; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - uint32_t i = 0, jobs_rx = 0; - int ret = -1; - uint8_t key[IMB_SHA_512_BLOCK_SIZE]; - uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; - DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - uint32_t key_len = 0; - size_t digest_len = 0; - size_t block_size = 0; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - switch (sha_type) { - case 224: - digest_len = vec->hmac_sha224_len; - block_size = IMB_SHA_256_BLOCK_SIZE; - break; - case 256: - digest_len = vec->hmac_sha256_len; - block_size = IMB_SHA_256_BLOCK_SIZE; - break; - case 384: - digest_len = vec->hmac_sha384_len; - block_size = IMB_SHA_384_BLOCK_SIZE; - break; - case 512: - digest_len = vec->hmac_sha512_len; - block_size = IMB_SHA_512_BLOCK_SIZE; - break; - default: - fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", - sha_type); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (vec->key_len <= block_size) { - memcpy(key, vec->key, vec->key_len); - key_len = (int) vec->key_len; - } else { - switch (sha_type) { - case 224: - IMB_SHA224(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; - break; - case 256: - IMB_SHA256(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; - break; - case 384: - IMB_SHA384(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; - break; - case 512: - IMB_SHA512(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; - break; - default: - fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", - sha_type); - goto end; - } - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - - switch (sha_type) { - case 224: - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 256: - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 384: - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 512: - default: - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - } - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - - switch (sha_type) { - case 224: - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 256: - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 384: - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 512: - default: - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - } - - /* empty the manager */ - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - job->enc_keys = NULL; - job->dec_keys = NULL; - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->dst = NULL; - job->key_len_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = digest_len; - job->iv = NULL; - job->iv_len_in_bytes = 0; - job->src = vec->data; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = 0; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; - job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; - job->cipher_mode = IMB_CIPHER_NULL; - - switch (sha_type) { - case 224: - job->hash_alg = IMB_AUTH_HMAC_SHA_224; - break; - case 256: - job->hash_alg = IMB_AUTH_HMAC_SHA_256; - break; - case 384: - job->hash_alg = IMB_AUTH_HMAC_SHA_384; - break; - case 512: - default: - job->hash_alg = IMB_AUTH_HMAC_SHA_512; - break; - } - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job) { - jobs_rx++; - /* - * SHANI HMAC-SHA implementation can return a completed - * job after 2nd submission - */ - if (num_jobs < 2) { - printf("%d Unexpected return from submit_job\n", - __LINE__); - goto end; - } - if (!hmac_shax_job_ok(vec, job, sha_type, - job->user_data, - padding, sizeof(padding))) - goto end; - } - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - jobs_rx++; - if (!hmac_shax_job_ok(vec, job, sha_type, - job->user_data, - padding, sizeof(padding))) - goto end; - } - - if (jobs_rx != num_jobs) { - printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); - goto end; - } - ret = 0; - - end: - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static int -test_hmac_shax_burst(struct IMB_MGR *mb_mgr, - const struct hmac_rfc4231_vector *vec, - const uint32_t num_jobs, - const int sha_type) -{ - struct IMB_JOB *job, *jobs[max_burst_jobs] = {NULL}; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - uint32_t i = 0, jobs_rx = 0, completed_jobs = 0; - int ret = -1, err; - uint8_t key[IMB_SHA_512_BLOCK_SIZE]; - uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; - DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - uint32_t key_len = 0; - size_t digest_len = 0; - size_t block_size = 0; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - switch (sha_type) { - case 224: - digest_len = vec->hmac_sha224_len; - block_size = IMB_SHA_256_BLOCK_SIZE; - break; - case 256: - digest_len = vec->hmac_sha256_len; - block_size = IMB_SHA_256_BLOCK_SIZE; - break; - case 384: - digest_len = vec->hmac_sha384_len; - block_size = IMB_SHA_384_BLOCK_SIZE; - break; - case 512: - digest_len = vec->hmac_sha512_len; - block_size = IMB_SHA_512_BLOCK_SIZE; - break; - default: - fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", - sha_type); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (vec->key_len <= block_size) { - memcpy(key, vec->key, vec->key_len); - key_len = (int) vec->key_len; - } else { - switch (sha_type) { - case 224: - IMB_SHA224(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; - break; - case 256: - IMB_SHA256(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; - break; - case 384: - IMB_SHA384(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; - break; - case 512: - IMB_SHA512(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; - break; - default: - fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", - sha_type); - goto end; - } - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - - switch (sha_type) { - case 224: - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 256: - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 384: - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 512: - default: - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - } - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - - switch (sha_type) { - case 224: - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 256: - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 384: - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 512: - default: - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - } - - while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) - IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); - - for (i = 0; i < num_jobs; i++) { - job = jobs[i]; - job->enc_keys = NULL; - job->dec_keys = NULL; - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->dst = NULL; - job->key_len_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = digest_len; - job->iv = NULL; - job->iv_len_in_bytes = 0; - job->src = vec->data; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = 0; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; - job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; - job->cipher_mode = IMB_CIPHER_NULL; - - switch (sha_type) { - case 224: - job->hash_alg = IMB_AUTH_HMAC_SHA_224; - break; - case 256: - job->hash_alg = IMB_AUTH_HMAC_SHA_256; - break; - case 384: - job->hash_alg = IMB_AUTH_HMAC_SHA_384; - break; - case 512: - default: - job->hash_alg = IMB_AUTH_HMAC_SHA_512; - break; - } - - job->user_data = auths[i]; - - } - - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); - err = imb_get_errno(mb_mgr); - - if (err != 0) { - printf("submit_burst error %d : '%s'\n", err, - imb_get_strerror(err)); - goto end; - } - - check_burst_jobs: - for (i = 0; i < completed_jobs; i++) { - job = jobs[i]; - - if (job->status != IMB_STATUS_COMPLETED) { - printf("job %u status not complete!\n", i+1); - goto end; - } - - if (!hmac_shax_job_ok(vec, job, sha_type, - job->user_data, - padding, sizeof(padding))) - goto end; - jobs_rx++; - } - - if (jobs_rx != num_jobs) { - completed_jobs = IMB_FLUSH_BURST(mb_mgr, - num_jobs - completed_jobs, - jobs); - if (completed_jobs == 0) { - printf("Expected %u jobs, received %u\n", - num_jobs, jobs_rx); - goto end; - } - goto check_burst_jobs; - } - ret = 0; - - end: - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static int -test_hmac_shax_hash_burst(struct IMB_MGR *mb_mgr, - const struct hmac_rfc4231_vector *vec, - const uint32_t num_jobs, - const int sha_type) -{ - struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; - uint8_t padding[16]; - uint8_t **auths = NULL; - uint32_t i = 0, jobs_rx = 0, completed_jobs = 0; - int ret = -1; - uint8_t key[IMB_SHA_512_BLOCK_SIZE]; - uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; - DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - uint32_t key_len = 0; - size_t digest_len = 0; - size_t block_size = 0; - - if (num_jobs == 0) - return 0; - - auths = malloc(num_jobs * sizeof(void *)); - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - switch (sha_type) { - case 224: - digest_len = vec->hmac_sha224_len; - block_size = IMB_SHA_256_BLOCK_SIZE; - break; - case 256: - digest_len = vec->hmac_sha256_len; - block_size = IMB_SHA_256_BLOCK_SIZE; - break; - case 384: - digest_len = vec->hmac_sha384_len; - block_size = IMB_SHA_384_BLOCK_SIZE; - break; - case 512: - digest_len = vec->hmac_sha512_len; - block_size = IMB_SHA_512_BLOCK_SIZE; - break; - default: - fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", - sha_type); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* prepare the key */ - memset(key, 0, sizeof(key)); - if (vec->key_len <= block_size) { - memcpy(key, vec->key, vec->key_len); - key_len = (int) vec->key_len; - } else { - switch (sha_type) { - case 224: - IMB_SHA224(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; - break; - case 256: - IMB_SHA256(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; - break; - case 384: - IMB_SHA384(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; - break; - case 512: - IMB_SHA512(mb_mgr, vec->key, vec->key_len, key); - key_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; - break; - default: - fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", - sha_type); - goto end; - } - } - - /* compute ipad hash */ - memset(buf, 0x36, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - - switch (sha_type) { - case 224: - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 256: - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 384: - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - case 512: - default: - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); - break; - } - - /* compute opad hash */ - memset(buf, 0x5c, sizeof(buf)); - for (i = 0; i < key_len; i++) - buf[i] ^= key[i]; - - switch (sha_type) { - case 224: - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 256: - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 384: - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - case 512: - default: - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); - break; - } - - for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; - job->enc_keys = NULL; - job->dec_keys = NULL; - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->dst = NULL; - job->key_len_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = digest_len; - job->iv = NULL; - job->iv_len_in_bytes = 0; - job->src = vec->data; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = 0; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; - job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; - job->cipher_mode = IMB_CIPHER_NULL; - - switch (sha_type) { - case 224: - job->hash_alg = IMB_AUTH_HMAC_SHA_224; - break; - case 256: - job->hash_alg = IMB_AUTH_HMAC_SHA_256; - break; - case 384: - job->hash_alg = IMB_AUTH_HMAC_SHA_384; - break; - case 512: - default: - job->hash_alg = IMB_AUTH_HMAC_SHA_512; - break; - } - - job->user_data = auths[i]; - - } - - completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, - job->hash_alg); - if (completed_jobs != num_jobs) { - int err = imb_get_errno(mb_mgr); - - if (err != 0) { - printf("submit_burst error %d : '%s'\n", err, - imb_get_strerror(err)); - goto end; - } else { - printf("submit_burst error: not enough " - "jobs returned!\n"); - goto end; - } - } - - for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; - - if (job->status != IMB_STATUS_COMPLETED) { - printf("job %u status not complete!\n", i+1); - goto end; - } - - if (!hmac_shax_job_ok(vec, job, sha_type, - job->user_data, - padding, sizeof(padding))) - goto end; - jobs_rx++; - } - - if (jobs_rx != num_jobs) { - printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); - goto end; - } - ret = 0; - - end: - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static void -test_hmac_shax_std_vectors(struct IMB_MGR *mb_mgr, - const int sha_type, - const uint32_t num_jobs, - struct test_suite_context *ts) -{ - const int vectors_cnt = DIM(hmac_sha256_sha512_vectors); - int vect; - - printf("HMAC-SHA%d standard test vectors (N jobs = %u):\n", - sha_type, num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; - const int flag = (sha_type == 224 && - hmac_sha256_sha512_vectors[idx].hmac_sha224 == NULL) || - (sha_type == 256 && - hmac_sha256_sha512_vectors[idx].hmac_sha256 == NULL) || - (sha_type == 384 && - hmac_sha256_sha512_vectors[idx].hmac_sha384 == NULL) || - (sha_type == 512 && - hmac_sha256_sha512_vectors[idx].hmac_sha512 == NULL); -#ifdef DEBUG - printf("[%d/%d] RFC4231 Test Case %d key_len:%d data_len:%d\n", - vect, vectors_cnt, - hmac_sha256_sha512_vectors[idx].test_case_num, - (int) hmac_sha256_sha512_vectors[idx].key_len, - (int) hmac_sha256_sha512_vectors[idx].data_len); -#else - printf("."); -#endif - - if (flag) { -#ifdef DEBUG - printf("Skipped vector %d, N/A for HMAC-SHA%d\n", - vect, sha_type); -#endif - continue; - } - - if (test_hmac_shax(mb_mgr, &hmac_sha256_sha512_vectors[idx], - num_jobs, sha_type)) { - printf("error #%d\n", vect); - test_suite_update(ts, 0, 1); - } else { - test_suite_update(ts, 1, 0); - } - if (test_hmac_shax_burst(mb_mgr, - &hmac_sha256_sha512_vectors[idx], - num_jobs, sha_type)) { - printf("error #%d - burst API\n", vect); - test_suite_update(ts, 0, 1); - } else { - test_suite_update(ts, 1, 0); - } - if (test_hmac_shax_hash_burst(mb_mgr, - &hmac_sha256_sha512_vectors[idx], - num_jobs, sha_type)) { - printf("error #%d - hash-only burst API\n", vect); - test_suite_update(ts, 0, 1); - } else { - test_suite_update(ts, 1, 0); - } - - } - printf("\n"); -} - -int -hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr) -{ - const int sha_types_tab[] = { - 224, 256, 384, 512 - }; - static const char * const sha_names_tab[] = { - "HMAC-SHA224", "HMAC-SHA256", "HMAC-SHA384", "HMAC-SHA512" - }; - unsigned i, num_jobs; - int errors = 0; - - for (i = 0; i < DIM(sha_types_tab); i++) { - struct test_suite_context ts; - - test_suite_start(&ts, sha_names_tab[i]); - for (num_jobs = 1; num_jobs <= max_burst_jobs; num_jobs++) - test_hmac_shax_std_vectors(mb_mgr, sha_types_tab[i], - num_jobs, &ts); - errors += test_suite_end(&ts); - } - - return errors; -} diff --git a/test/wycheproof/aead_test.h b/test/include/aead_test.h similarity index 97% rename from test/wycheproof/aead_test.h rename to test/include/aead_test.h index dd2f45d563f133c33974eab6ce4f6de04ea9bc0a..995947586950bb9947af94d5d9e728932edbefa3 100644 --- a/test/wycheproof/aead_test.h +++ b/test/include/aead_test.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/mac_test.h b/test/include/mac_test.h similarity index 97% rename from test/wycheproof/mac_test.h rename to test/include/mac_test.h index 89b71f7e60f2dad1e6a8886794268c048919493e..58056fac64aa63d45b401d261c75d8349931f47e 100644 --- a/test/wycheproof/mac_test.h +++ b/test/include/mac_test.h @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/utils.h b/test/include/utils.h similarity index 97% rename from test/utils.h rename to test/include/utils.h index bd71380c340f44321f99c1156d3c926a13d7facd..322bf5d4f4497c4302cd6f691713c11ea94d2f2b 100644 --- a/test/utils.h +++ b/test/include/utils.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -34,6 +34,8 @@ #define DIM(_x) (sizeof(_x)/sizeof(_x[0])) #define DIV_ROUND_UP(x, y) ((x + y - 1) / y) +extern int quiet_mode; + void hexdump(FILE *fp, const char *msg, const void *p, size_t len); void hexdump_ex(FILE *fp, const char *msg, const void *p, size_t len, const void *start_ptr); diff --git a/test/kat-app/CMakeLists.txt b/test/kat-app/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a96c23705347bb776255c1cf917235ea8e00eafe --- /dev/null +++ b/test/kat-app/CMakeLists.txt @@ -0,0 +1,203 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +######################################## +# set app and library names +######################################## +set(TEST_APP imb-kat) +set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib) +set(TEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include) + +######################################## +# set imb-kat source files +######################################## +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(TEST_APP_SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/main.c + ${CMAKE_CURRENT_SOURCE_DIR}/api_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/zuc_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/snow3g_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/direct_api_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/clear_mem_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/direct_api_param_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.c + ) +else() + set(TEST_APP_SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/main.c + ${CMAKE_CURRENT_SOURCE_DIR}/gcm_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/ctr_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/customop_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/des_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/ccm_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/cmac_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha1_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha256_sha512_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_md5_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/aes_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/sha_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/chained_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/api_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/pon_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/ecb_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/zuc_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/kasumi_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/snow3g_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/direct_api_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/clear_mem_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/hec_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/xcbc_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/aes_cbcs_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/crc_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/chacha_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/poly1305_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/chacha20_poly1305_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/null_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/snow_v_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/direct_api_param_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/quic_ecb_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/gmac_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/ghash_test.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha1.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha224.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha256.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha384.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha512.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_md5.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/gmac_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/ghash_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/poly1305_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/cmac_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/xcbc_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/sha_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.c + ) +endif() + + +######################################## +# set C compiler and NASM options +######################################## +# add OS specific options +if(WINDOWS) + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/windows.cmake) +else() + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/unix.cmake) +endif() + +######################################## +# add targets +######################################## +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${LIB_DIR} ${TEST_INCLUDE_DIR}) +link_directories(${LIB_DIR}) + +add_executable(${TEST_APP} ${TEST_APP_SRC_FILES}) +target_link_libraries(${TEST_APP} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${TEST_APP} PRIVATE ${APP_DEFINES}) + +######################################## +# add tests +######################################## + +# set working directory for tests +if(IMB_BIN_DIR) + set(TEST_APP_BIN_DIR "${IMB_BIN_DIR}") +else() + set(TEST_APP_BIN_DIR "${CMAKE_CURRENT_BINARY_DIR}") +endif() + +# append config type for multi-config generators +get_property(multi_config_gen GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) +if (multi_config_gen) + STRING(APPEND TEST_APP_BIN_DIR "/$,Debug,Release>") +endif() + +# set arch options +set(ARCH_SSE "--no-avx --no-avx2 --no-avx512") +set(ARCH_AVX "--no-sse --no-avx2 --no-avx512") +set(ARCH_AVX2 "--no-sse --no-avx --no-avx512") +set(ARCH_AVX512 "--no-sse --no-avx --no-avx2") +set(ARCH_AARCH64 "--no-sve256") +set(ARCH_SVE256 "--no-aarch64") + +# check for SDE +if (WINDOWS) + find_program(SDE NAMES sde) +else() + find_program(SDE NAMES sde64) +endif() + +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + + add_test(NAME KAT-AARCH64 + COMMAND ${TEST_APP} ${ARCH_AARCH64} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-SVE256 + COMMAND ${TEST_APP} ${ARCH_SVE256} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + +else() + + add_test(NAME KAT-SSE-T1 + COMMAND ${TEST_APP} ${ARCH_SSE} --shani-off + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-SSE-T2 + COMMAND ${TEST_APP} ${ARCH_SSE} --gfni-off + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-SSE-T3 + COMMAND ${TEST_APP} ${ARCH_SSE} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-AVX-T1 + COMMAND ${TEST_APP} ${ARCH_AVX} --shani-off + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-AVX-T2 + COMMAND ${TEST_APP} ${ARCH_AVX} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-AVX2-T1 + COMMAND ${TEST_APP} ${ARCH_AVX2} --shani-off --gfni-off + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-AVX2-T2 + COMMAND ${TEST_APP} ${ARCH_AVX2} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-AVX512-T1 + COMMAND ${TEST_APP} ${ARCH_AVX512} --shani-off --gfni-off + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + add_test(NAME KAT-AVX512-T2 + COMMAND ${TEST_APP} ${ARCH_AVX512} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + + # SDE tests + if(SDE) + add_test(NAME KAT-AVX2-T3 + COMMAND ${SDE} -cmt -- ./${TEST_APP} ${ARCH_AVX2} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) + endif() + +endif() diff --git a/test/kat-app/Makefile b/test/kat-app/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..af47ec8ae979c3c60159990db367644fb8c89f31 --- /dev/null +++ b/test/kat-app/Makefile @@ -0,0 +1,68 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP := imb-kat + +# include common options +include ../common/common.mk + +# imb-kat modules +ifneq ($(ARCH),aarch64) +SOURCES := main.c gcm_test.c ctr_test.c customop_test.c des_test.c ccm_test.c \ + cmac_test.c hmac_sha1_test.c hmac_sha256_sha512_test.c \ + hmac_md5_test.c aes_test.c sha_test.c chained_test.c api_test.c pon_test.c \ + ecb_test.c zuc_test.c kasumi_test.c snow3g_test.c direct_api_test.c clear_mem_test.c \ + hec_test.c xcbc_test.c aes_cbcs_test.c crc_test.c chacha_test.c poly1305_test.c \ + chacha20_poly1305_test.c null_test.c snow_v_test.c direct_api_param_test.c quic_ecb_test.c \ + hmac_sha1.json.c hmac_sha224.json.c hmac_sha256.json.c hmac_sha384.json.c hmac_sha512.json.c \ + hmac_md5.json.c gmac_test.json.c ghash_test.c ghash_test.json.c poly1305_test.json.c \ + cmac_test.json.c xcbc_test.json.c sha_test.json.c gmac_test.c +else # aarch64 +SOURCES := main.c api_test.c zuc_test.c snow3g_test.c direct_api_test.c \ + clear_mem_test.c direct_api_param_test.c +endif # aarch64 + +OBJECTS := $(SOURCES:%.c=%.o) utils.o + +# targets come here +all: $(APP) + +$(APP): $(OBJECTS) + $(CC) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +utils.o: ../common/utils.c + $(CC) -c $(CFLAGS) $< -o $@ + +.PHONY: clean +clean: + -rm -f $(OBJECTS) $(APP) $(DEP_FILES) + +# if target not clean then make dependencies +ifneq ($(MAKECMDGOALS),clean) +-include $(wildcard *.d) +endif + diff --git a/test/kat-app/README.md b/test/kat-app/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c0be7ec656a93859ae57beef73d38a6f2911cd48 --- /dev/null +++ b/test/kat-app/README.md @@ -0,0 +1,45 @@ +# Intel(R) Multi-Buffer Crypto for IPsec Library - Known Answer Test (KAT) Application + +## Contents + +- Overview +- Usage + + +## Overview + +The KAT application validates correct algorithm implementations by passing +predefined inputs to the library API and verifying the generated output against a +known correct result. By default, the application will test all algorithms across all +architectures e.g. SSE, AVX, AVX2, AVX512 and prints overall test result "PASS" or "FAIL". + +## Usage + +Before running the application, ensure the library is installed by following the instructions +in the [README](https://github.com/intel/intel-ipsec-mb/tree/main/test#library-installation). + +### Linux +To test all algorithms across all architectures: +`./imb-kat` + +To test all algorithms on SSE architecture only: +`./imb-kat --no-avx --no-avx2 --no-avx512` + +To test AES-GCM on AVX512 architecture only: +`./imb-kat --no-sse --no-avx --no-avx2 --test-type GCM` + +To display an extensive help page: +`./imb-kat --help` + +### Windows +To test all algorithms across all architectures: +`imb-kat.exe` + +To test all algorithms on SSE architecture only: +`imb-kat.exe --no-avx --no-avx2 --no-avx512` + +To test AES-GCM on AVX512 architecture only: +`imb-kat.exe --no-sse --no-avx --no-avx2 --test-type GCM` + +To display an extensive help page: +`imb-kat.exe --help` diff --git a/test/aes_cbcs_test.c b/test/kat-app/aes_cbcs_test.c similarity index 99% rename from test/aes_cbcs_test.c rename to test/kat-app/aes_cbcs_test.c index 07bed0e046977aa3a259e1839afe4d6882dbda51..4b5c5ded3ab8bbbcb309dc4724b01fa91349e233 100644 --- a/test/aes_cbcs_test.c +++ b/test/kat-app/aes_cbcs_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -3642,15 +3642,19 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); - printf("%s (N jobs = %d):\n", banner, num_jobs); + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); for (vect = 0; vect < vec_cnt; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("[%d/%d] Standard vector key_len:%d\n", - vect + 1, vec_cnt, - (int) vec_tab[vect].Klen); + printf("[%d/%d] Standard vector key_len:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].Klen); #else - printf("."); + printf("."); #endif + } + IMB_AES_KEYEXP_128(mb_mgr, vec_tab[vect].K, enc_keys, dec_keys); @@ -3706,7 +3710,8 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } int diff --git a/test/aes_test.c b/test/kat-app/aes_test.c similarity index 91% rename from test/aes_test.c rename to test/kat-app/aes_test.c index ca2ed4b2a575073ee91ba1754b5ec71ca50b193c..6b4cc3902755c099b37db9273895f09fe078ebfa 100644 --- a/test/aes_test.c +++ b/test/kat-app/aes_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -922,6 +922,8 @@ static const uint8_t CFBC2[] = { * Test vectors from * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a.pdf */ + +/* 128-Bit */ static const uint8_t CFBK3[] = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c @@ -989,13 +991,116 @@ static const uint8_t CFBC6[] = { 0xc0, 0x4b, 0x05, 0x35, 0x7c, 0x5d, 0x1c, 0x0e, 0xea, 0xc4, 0xc6, 0x6f, 0x9f, 0xf7, 0xf2, 0xe6 }; -static struct aes_vector aes_cfb_128_tab[] = { - {CFBK1, CFBIV1, CFBP1, sizeof(CFBP1), CFBC1, sizeof(CFBK1)}, - {CFBK2, CFBIV2, CFBP2, sizeof(CFBP2), CFBC2, sizeof(CFBK2)}, - {CFBK3, CFBIV3, CFBP3, sizeof(CFBP3), CFBC3, sizeof(CFBK3)}, - {CFBK4, CFBIV4, CFBP4, sizeof(CFBP4), CFBC4, sizeof(CFBK4)}, - {CFBK5, CFBIV5, CFBP5, sizeof(CFBP5), CFBC5, sizeof(CFBK5)}, - {CFBK6, CFBIV6, CFBP6, sizeof(CFBP6), CFBC6, sizeof(CFBK6)}, + +static const uint8_t CFBK7[] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c +}; +static const uint8_t CFBIV7[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const uint8_t CFBP7[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96 +}; +static const uint8_t CFBC7[] = { + 0x3b, 0x3f, 0xd9, 0x2e, 0xb7, 0x2d, 0xad, 0x20 +}; +static const uint8_t CFBK8[] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c +}; +static const uint8_t CFBIV8[] = { + 0xc8, 0xa6, 0x45, 0x37, 0xa0, 0xb3, 0xa9, 0x3f, + 0xcd, 0xe3, 0xcd, 0xad, 0x9f, 0x1c, 0xe5, 0x8b +}; +static const uint8_t CFBP8[] = { + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11 +}; +static const uint8_t CFBC8[] = { + 0x26, 0x75, 0x1f, 0x67, 0xa3, 0xcb, 0xb1, 0x40 +}; + +/* 256-Bit */ +static const uint8_t CFBK9[] = { + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, + 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, + 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 +}; +static const uint8_t CFBIV9[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const uint8_t CFBP9[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a +}; +static const uint8_t CFBC9[] = { + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, + 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60 +}; +static const uint8_t CFBIV10[] = { + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, + 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60 +}; +static const uint8_t CFBP10[] = { + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51 +}; +static const uint8_t CFBC10[] = { + 0x39, 0xff, 0xed, 0x14, 0x3b, 0x28, 0xb1, 0xc8, + 0x32, 0x11, 0x3c, 0x63, 0x31, 0xe5, 0x40, 0x7b +}; +static const uint8_t CFBIV11[] = { + 0x39, 0xff, 0xed, 0x14, 0x3b, 0x28, 0xb1, 0xc8, + 0x32, 0x11, 0x3c, 0x63, 0x31, 0xe5, 0x40, 0x7b +}; +static const uint8_t CFBP11[] = { + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef +}; +static const uint8_t CFBC11[] = { + 0xdf, 0x10, 0x13, 0x24, 0x15, 0xe5, 0x4b, 0x92, + 0xa1, 0x3e, 0xd0, 0xa8, 0x26, 0x7a, 0xe2, 0xf9 +}; +static const uint8_t CFBIV12[] = { + 0xdf, 0x10, 0x13, 0x24, 0x15, 0xe5, 0x4b, 0x92, + 0xa1, 0x3e, 0xd0, 0xa8, 0x26, 0x7a, 0xe2, 0xf9 +}; +static const uint8_t CFBP12[] = { + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; +static const uint8_t CFBC12[] = { + 0x75, 0xa3, 0x85, 0x74, 0x1a, 0xb9, 0xce, 0xf8, + 0x20, 0x31, 0x62, 0x3d, 0x55, 0xb1, 0xe4, 0x71 +}; +static const uint8_t CFBIV13[] = { + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, + 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60 +}; +static const uint8_t CFBP13[] = { + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c +}; +static const uint8_t CFBC13[] = { + 0x39, 0xff, 0xed, 0x14, 0x3b, 0x28, 0xb1, 0xc8 +}; + +static struct aes_vector aes_cfb_tab[] = { + {CFBK1, CFBIV1, CFBP1, sizeof(CFBP1), CFBC1, sizeof(CFBK1)}, + {CFBK2, CFBIV2, CFBP2, sizeof(CFBP2), CFBC2, sizeof(CFBK2)}, + {CFBK3, CFBIV3, CFBP3, sizeof(CFBP3), CFBC3, sizeof(CFBK3)}, + {CFBK4, CFBIV4, CFBP4, sizeof(CFBP4), CFBC4, sizeof(CFBK4)}, + {CFBK5, CFBIV5, CFBP5, sizeof(CFBP5), CFBC5, sizeof(CFBK5)}, + {CFBK6, CFBIV6, CFBP6, sizeof(CFBP6), CFBC6, sizeof(CFBK6)}, + {CFBK7, CFBIV7, CFBP7, sizeof(CFBP7), CFBC7, sizeof(CFBK7)}, + {CFBK8, CFBIV8, CFBP8, sizeof(CFBP8), CFBC8, sizeof(CFBK8)}, + {CFBK9, CFBIV9, CFBP9, sizeof(CFBP9), CFBC9, sizeof(CFBK9)}, + {CFBK9, CFBIV10, CFBP10, sizeof(CFBP10), CFBC10, sizeof(CFBK9)}, + {CFBK9, CFBIV11, CFBP11, sizeof(CFBP11), CFBC11, sizeof(CFBK9)}, + {CFBK9, CFBIV12, CFBP12, sizeof(CFBP12), CFBC12, sizeof(CFBK9)}, + {CFBK9, CFBIV13, CFBP13, sizeof(CFBP13), CFBC13, sizeof(CFBK9)}, }; /* =================================================================== */ @@ -2039,6 +2144,8 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, job->msg_len_to_cipher_in_bytes = text_len; job->user_data = targets[i]; job->user_data2 = (void *)((uint64_t)i); + + imb_set_session(mb_mgr, job); } completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); @@ -2211,16 +2318,21 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); - printf("%s (N jobs = %d):\n", banner, num_jobs); + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); for (vect = 0; vect < vec_cnt; vect++) { struct test_suite_context *ctx; + + if (!quiet_mode) { #ifdef DEBUG - printf("[%d/%d] Standard vector key_len:%d\n", - vect + 1, vec_cnt, - (int) vec_tab[vect].Klen); + printf("[%d/%d] Standard vector key_len:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].Klen); #else - printf("."); + printf("."); #endif + } + switch (vec_tab[vect].Klen) { case 16: IMB_AES_KEYEXP_128(mb_mgr, vec_tab[vect].K, enc_keys, @@ -2401,7 +2513,8 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } static int @@ -2596,27 +2709,28 @@ test_docrc_vectors(struct IMB_MGR *mb_mgr, DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); - printf("%s (N jobs = %d):\n", banner, num_jobs); + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); for (vect = 0; vect < vec_cnt; vect++) { struct test_suite_context *ctx; + + if (!quiet_mode) { #ifdef DEBUG - printf("[%d/%d] Standard vector\n", - vect + 1, vec_cnt); + printf("[%d/%d] Standard vector\n", + vect + 1, vec_cnt); #else - printf("."); + printf("."); #endif - switch (vec_tab[vect].key_len) { - case 16: + } + + if (vec_tab[vect].key_len == 16) { IMB_AES_KEYEXP_128(mb_mgr, vec_tab[vect].key, enc_keys, dec_keys); ctx = ctx128; - break; - case 32: - default: + } else { IMB_AES_KEYEXP_256(mb_mgr, vec_tab[vect].key, enc_keys, dec_keys); ctx = ctx256; - break; } @@ -2641,90 +2755,145 @@ test_docrc_vectors(struct IMB_MGR *mb_mgr, } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } static int -cfb128_validate_ok(const uint8_t *output, const uint8_t *in_text, - const size_t plen, const unsigned i, const unsigned is_enc, - const int in_place) +cfb_validate_ok(const uint8_t *output, const uint8_t *in_text, + const size_t plen, const uint32_t klen, + const unsigned i, const unsigned is_enc, + const int in_place) { if (memcmp(output, in_text, plen) != 0) { - printf("\nAES-CFB128 standard test vector %u %s (%s): fail\n", + printf("\nAES-CFB%s standard test vector %u %s (%s): fail\n", + (klen == 16) ? "128" : "256", i + 1, (is_enc) ? "encrypt" : "decrypt", (in_place) ? "in-place" : "out-of-place"); return 0; } + #ifdef DEBUG - printf("Standard test vector %u %s %s\n", i + 1, - (in_place) ? "in-place" : "out-of-place", - (is_enc) ? "encrypt" : "decrypt"); -#else - printf("."); + if (!quiet_mode) { + printf("Standard test vector %u %s %s\n", i + 1, + (in_place) ? "in-place" : "out-of-place", + (is_enc) ? "encrypt" : "decrypt"); + } #endif - return 1; } - static int -cfb128_validate(struct IMB_MGR *mb_mgr) +cfb_validate(struct IMB_MGR *mb_mgr, + const struct aes_vector *p_vec, + const unsigned i) { - unsigned i; - - printf("AES-CFB128 standard test vectors:\n"); - for (i = 0; i < DIM(aes_cfb_128_tab); i++) { - uint8_t output1[16]; - uint8_t output2[16]; - DECLARE_ALIGNED(uint32_t key[4], 16); - DECLARE_ALIGNED(uint32_t keys_enc[11*4], 16); - DECLARE_ALIGNED(uint32_t keys_dec[11*4], 16); - - memcpy(key, aes_cfb_128_tab[i].K, aes_cfb_128_tab[i].Klen); - IMB_AES_KEYEXP_128(mb_mgr, key, keys_enc, keys_dec); - - /* Out of place */ - - /* encrypt test */ - IMB_AES128_CFB_ONE(mb_mgr, output1, aes_cfb_128_tab[i].P, - aes_cfb_128_tab[i].IV, keys_enc, - aes_cfb_128_tab[i].Plen); - if (!cfb128_validate_ok(output1, aes_cfb_128_tab[i].C, - aes_cfb_128_tab[i].Plen, i, 1, 0)) - return 0; + uint8_t output1[16]; + uint8_t output2[16]; + const uint32_t kLength = p_vec->Klen; + DECLARE_ALIGNED(uint32_t keys_enc[15*4], 16); + DECLARE_ALIGNED(uint32_t keys_dec[15*4], 16); + + if (kLength == 16) + IMB_AES_KEYEXP_128(mb_mgr, p_vec->K, + keys_enc, keys_dec); + else + IMB_AES_KEYEXP_256(mb_mgr, p_vec->K, + keys_enc, keys_dec); + /* Out of place */ + + /* encrypt test */ + if (kLength == 16) + IMB_AES128_CFB_ONE(mb_mgr, output1, p_vec->P, + p_vec->IV, keys_enc, + p_vec->Plen); + else + IMB_AES256_CFB_ONE(mb_mgr, output1, p_vec->P, + p_vec->IV, keys_enc, + p_vec->Plen); + if (!cfb_validate_ok(output1, p_vec->C, + p_vec->Plen, p_vec->Klen, i, 1, 0)) + return 0; - /* decrypt test */ + /* decrypt test */ + if (kLength == 16) IMB_AES128_CFB_ONE(mb_mgr, output2, output1, - aes_cfb_128_tab[i].IV, keys_enc, - aes_cfb_128_tab[i].Plen); - if (!cfb128_validate_ok(output2, aes_cfb_128_tab[i].P, - aes_cfb_128_tab[i].Plen, i, 0, 0)) - return 0; - - /* In place */ + p_vec->IV, keys_enc, + p_vec->Plen); + else + IMB_AES256_CFB_ONE(mb_mgr, output2, output1, + p_vec->IV, keys_enc, + p_vec->Plen); + if (!cfb_validate_ok(output2, p_vec->P, + p_vec->Plen, p_vec->Klen, i, 0, 0)) + return 0; + /* In place */ - /* encrypt test */ - memcpy(output1, aes_cfb_128_tab[i].P, aes_cfb_128_tab[i].Plen); + /* encrypt test */ + memcpy(output1, p_vec->P, p_vec->Plen); + if (kLength == 16) IMB_AES128_CFB_ONE(mb_mgr, output1, output1, - aes_cfb_128_tab[i].IV, keys_enc, - aes_cfb_128_tab[i].Plen); - if (!cfb128_validate_ok(output1, aes_cfb_128_tab[i].C, - aes_cfb_128_tab[i].Plen, i, 1, 1)) - return 0; + p_vec->IV, keys_enc, + p_vec->Plen); + else + IMB_AES256_CFB_ONE(mb_mgr, output1, output1, + p_vec->IV, keys_enc, + p_vec->Plen); + if (!cfb_validate_ok(output1, p_vec->C, + p_vec->Plen, p_vec->Klen, i, 1, 1)) + return 0; - /* decrypt test */ - memcpy(output1, aes_cfb_128_tab[i].C, aes_cfb_128_tab[i].Plen); + /* decrypt test */ + memcpy(output1, p_vec->C, p_vec->Plen); + if (kLength == 16) IMB_AES128_CFB_ONE(mb_mgr, output1, output1, - aes_cfb_128_tab[i].IV, keys_enc, - aes_cfb_128_tab[i].Plen); - if (!cfb128_validate_ok(output1, aes_cfb_128_tab[i].P, - aes_cfb_128_tab[i].Plen, i, 0, 1)) - return 0; - } - printf("\n"); + p_vec->IV, keys_enc, + p_vec->Plen); + else + IMB_AES256_CFB_ONE(mb_mgr, output1, output1, + p_vec->IV, keys_enc, + p_vec->Plen); + if (!cfb_validate_ok(output1, p_vec->P, + p_vec->Plen, p_vec->Klen, i, 0, 1)) + return 0; return 1; } +static void +cfb_test_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx128, + struct test_suite_context *ctx256, const int num_jobs) +{ + unsigned vect; + + if (!quiet_mode) + printf("AES-CFB test vectors (N jobs = %d):\n", num_jobs); + for (vect = 0; vect < DIM(aes_cfb_tab); vect++) { + struct test_suite_context *ctx; + + if (!quiet_mode) { +#ifdef DEBUG + printf("[%d/%lu] Standard %s-bit vector\n", + vect + 1, + (unsigned long) DIM(aes_cfb_tab), + (aes_cfb_tab[vect].Klen == 16) ? "128" : "256"); +#else + printf("."); +#endif + } + + if (aes_cfb_tab[vect].Klen == 16) + ctx = ctx128; + else + ctx = ctx256; + if (!cfb_validate(mb_mgr, &aes_cfb_tab[vect], vect)) + test_suite_update(ctx, 0, 1); + else + test_suite_update(ctx, 1, 0); + } + if (!quiet_mode) + printf("\n"); +} + int aes_test(struct IMB_MGR *mb_mgr) { @@ -2763,8 +2932,13 @@ aes_test(struct IMB_MGR *mb_mgr) errors += test_suite_end(&ctx128); errors += test_suite_end(&ctx256); - if (!cfb128_validate(mb_mgr)) - errors++; + test_suite_start(&ctx128, "AES-CFB-128"); + test_suite_start(&ctx256, "AES-CFB-256"); + for (i = 0; i < DIM(num_jobs_tab); i++) + cfb_test_vectors(mb_mgr, &ctx128, &ctx256, num_jobs_tab[i]); + errors += test_suite_end(&ctx128); + errors += test_suite_end(&ctx256); + test_suite_start(&ctx128, "DOCSIS-SEC-128-CRC32"); test_suite_start(&ctx256, "DOCSIS-SEC-256-CRC32"); diff --git a/test/api_test.c b/test/kat-app/api_test.c similarity index 84% rename from test/api_test.c rename to test/kat-app/api_test.c index 414a00395ece2217dd4ec9ac1a3a0e7edf45bb39..f581408df65923b20fb41a1050a6cdadb30cc9e0 100644 --- a/test/api_test.c +++ b/test/kat-app/api_test.c @@ -1,15 +1,15 @@ /***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation + Copyright (c) 2018-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors + * Neither the name of Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -43,38 +43,44 @@ int api_test(struct IMB_MGR *mb_mgr); enum { - TEST_UNEXPECTED_JOB = 1, - TEST_INVALID_JOB, - TEST_INVALID_BURST, - TEST_AUTH_SRC_NULL = 100, - TEST_AUTH_AUTH_TAG_OUTPUT_NULL, - TEST_AUTH_TAG_OUTPUT_LEN_ZERO, - TEST_AUTH_MSG_LEN_ZERO, - TEST_AUTH_MSG_LEN_GT_MAX, - TEST_AUTH_IV_LEN, - TEST_AUTH_NULL_HMAC_OPAD, - TEST_AUTH_NULL_HMAC_IPAD, - TEST_AUTH_NULL_XCBC_K1_EXP, - TEST_AUTH_NULL_XCBC_K2, - TEST_AUTH_NULL_XCBC_K3, - TEST_AUTH_NULL_GHASH_KEY, - TEST_AUTH_NULL_GHASH_INIT_TAG, - TEST_AUTH_NULL_GMAC_KEY, - TEST_AUTH_NULL_GMAC_IV, - TEST_AUTH_GMAC_IV_LEN, - TEST_CIPH_SRC_NULL = 200, - TEST_CIPH_DST_NULL, - TEST_CIPH_IV_NULL, - TEST_CIPH_ENC_KEY_NULL, - TEST_CIPH_DEC_KEY_NULL, - TEST_CIPH_MSG_LEN_ZERO, - TEST_CIPH_MSG_LEN_GT_MAX, - TEST_CIPH_NEXT_IV_NULL, - TEST_CIPH_IV_LEN, - TEST_CIPH_DIR, - TEST_INVALID_PON_PLI = 300, + TEST_UNEXPECTED_JOB = 1, + TEST_INVALID_JOB, + TEST_INVALID_BURST, + TEST_AUTH_SRC_NULL = 100, + TEST_AUTH_AUTH_TAG_OUTPUT_NULL, + TEST_AUTH_TAG_OUTPUT_LEN_ZERO, + TEST_AUTH_MSG_LEN_ZERO, + TEST_AUTH_MSG_LEN_GT_MAX, + TEST_AUTH_IV_LEN, + TEST_AUTH_NULL_HMAC_OPAD, + TEST_AUTH_NULL_HMAC_IPAD, + TEST_AUTH_NULL_XCBC_K1_EXP, + TEST_AUTH_NULL_XCBC_K2, + TEST_AUTH_NULL_XCBC_K3, + TEST_AUTH_NULL_GHASH_KEY, + TEST_AUTH_NULL_GHASH_INIT_TAG, + TEST_AUTH_NULL_GMAC_KEY, + TEST_AUTH_NULL_GMAC_IV, + TEST_AUTH_GMAC_IV_LEN, + TEST_CIPH_SRC_NULL = 200, + TEST_CIPH_DST_NULL, + TEST_CIPH_IV_NULL, + TEST_CIPH_ENC_KEY_NULL, + TEST_CIPH_DEC_KEY_NULL, + TEST_CIPH_MSG_LEN_ZERO, + TEST_CIPH_MSG_LEN_GT_MAX, + TEST_CIPH_NEXT_IV_NULL, + TEST_CIPH_IV_LEN, + TEST_CIPH_DIR, + TEST_INVALID_PON_PLI = 300, }; +static void print_progress(void) +{ + if (!quiet_mode) + printf("."); +} + /* * @brief Performs JOB API behavior tests */ @@ -93,14 +99,14 @@ test_job_api(struct IMB_MGR *mb_mgr) __func__, TEST_UNEXPECTED_JOB); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != 0) { printf("%s: test %d, unexpected error: %s\n", __func__, TEST_UNEXPECTED_JOB, imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); /* ======== test 2 : invalid cipher and mac */ memset(job, 0, sizeof(*job)); @@ -111,14 +117,14 @@ test_job_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_JOB); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err == 0) { printf("%s: test %d, unexpected error: %s\n", __func__, TEST_INVALID_JOB, imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); if (job_next->status != IMB_STATUS_INVALID_ARGS) { /* Invalid job is returned, and status should be INVALID_ARGS */ @@ -126,7 +132,7 @@ test_job_api(struct IMB_MGR *mb_mgr) "IMB_STATUS_INVALID_ARGS\n", __func__, TEST_INVALID_JOB); return 1; } - printf("."); + print_progress(); job_next = IMB_GET_NEXT_JOB(mb_mgr); if (job == job_next) { @@ -135,14 +141,14 @@ test_job_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_JOB); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != 0) { printf("%s: test %d, unexpected error: %s\n", __func__, TEST_INVALID_JOB, imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); job = IMB_GET_COMPLETED_JOB(mb_mgr); if (job) { @@ -151,20 +157,21 @@ test_job_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_JOB); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != 0) { printf("%s: test %d, unexpected error: %s\n", __func__, TEST_INVALID_JOB, imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); /* clean up */ while (IMB_FLUSH_JOB(mb_mgr) != NULL) ; - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -269,6 +276,7 @@ fill_in_job(struct IMB_JOB *job, job->auth_tag_output_len_in_bytes = tag_len_tab[job->hash_alg]; switch (job->cipher_mode) { +#ifndef __aarch64__ case IMB_CIPHER_CBC: case IMB_CIPHER_CBCS_1_9: job->key_len_in_bytes = UINT64_C(16); @@ -281,8 +289,10 @@ fill_in_job(struct IMB_JOB *job, job->key_len_in_bytes = UINT64_C(16); job->iv_len_in_bytes = UINT64_C(16); break; +#endif /* __aarch64__ */ case IMB_CIPHER_NULL: break; +#ifndef __aarch64__ case IMB_CIPHER_DOCSIS_SEC_BPI: /* it has to be set regardless of direction (AES-CFB) */ job->key_len_in_bytes = UINT64_C(16); @@ -330,12 +340,12 @@ fill_in_job(struct IMB_JOB *job, uint64_t *ptr64 = (uint64_t *) dust_bin; ptr64[0] = ((pli >> 8) & 0xff) | - ((pli & 0xff) << 8); + ((pli & 0xff) << 8); break; case IMB_CIPHER_ECB: job->key_len_in_bytes = UINT64_C(16); - job->iv_len_in_bytes = 0; break; +#endif /* __aarch64__ */ case IMB_CIPHER_ZUC_EEA3: job->key_len_in_bytes = UINT64_C(16); job->iv_len_in_bytes = 16; @@ -344,6 +354,7 @@ fill_in_job(struct IMB_JOB *job, job->key_len_in_bytes = UINT64_C(16); job->iv_len_in_bytes = 16; break; +#ifndef __aarch64__ case IMB_CIPHER_KASUMI_UEA1_BITLEN: job->key_len_in_bytes = UINT64_C(16); job->iv_len_in_bytes = 8; @@ -377,11 +388,13 @@ fill_in_job(struct IMB_JOB *job, job->key_len_in_bytes = UINT64_C(16); job->iv_len_in_bytes = UINT64_C(12); break; +#endif /* __aarch64__ */ default: break; } switch (job->hash_alg) { +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: case IMB_AUTH_HMAC_SHA_224: case IMB_AUTH_HMAC_SHA_256: @@ -408,8 +421,10 @@ fill_in_job(struct IMB_JOB *job, case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: case IMB_AUTH_CRC7_FP_HEADER: case IMB_AUTH_CRC6_IUUP_HEADER: +#endif /* __aarch64__ */ case IMB_AUTH_NULL: break; +#ifndef __aarch64__ case IMB_AUTH_AES_XCBC: job->u.XCBC._k1_expanded = (const uint32_t *) dust_bin; job->u.XCBC._k2 = dust_bin; @@ -456,12 +471,14 @@ fill_in_job(struct IMB_JOB *job, job->key_len_in_bytes = 16; job->iv_len_in_bytes = 16; break; +#endif /* __aarch64__*/ case IMB_AUTH_ZUC_EIA3_BITLEN: case IMB_AUTH_ZUC256_EIA3_BITLEN: job->u.ZUC_EIA3._key = dust_bin; job->u.ZUC_EIA3._iv = dust_bin; job->auth_tag_output_len_in_bytes = 4; break; +#ifndef __aarch64__ case IMB_AUTH_DOCSIS_CRC32: job->auth_tag_output_len_in_bytes = 4; job->hash_start_src_offset_in_bytes = 32; @@ -475,12 +492,14 @@ fill_in_job(struct IMB_JOB *job, job->key_len_in_bytes = UINT64_C(16); job->iv_len_in_bytes = UINT64_C(16); break; +#endif /* __aarch64__ */ case IMB_AUTH_SNOW3G_UIA2_BITLEN: job->msg_len_to_hash_in_bits = msg_len_to_hash * 8; job->u.SNOW3G_UIA2._key = dust_bin; job->u.SNOW3G_UIA2._iv = dust_bin; job->auth_tag_output_len_in_bytes = 4; break; +#ifndef __aarch64__ case IMB_AUTH_KASUMI_UIA1: job->u.KASUMI_UIA1._key = dust_bin; job->auth_tag_output_len_in_bytes = 4; @@ -534,6 +553,7 @@ fill_in_job(struct IMB_JOB *job, job->iv_len_in_bytes = 16; job->auth_tag_output_len_in_bytes = 16; break; +#endif /* __aarch64__ */ default: break; } @@ -643,8 +663,6 @@ static int is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, const int test_num, int expected_errnum) { - // To do: it's not implemented on Arm platform -#ifdef __x86_64__ IMB_JOB * jobs[MAX_BURST_JOBS] = {NULL}; uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; int err; @@ -656,7 +674,7 @@ is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, for (i = 0; i < n_jobs; i++) *jobs[i] = *job; - /* submit the job for processing */ + /* submit the job for processing */ completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); if (completed_jobs != 0) { printf("%s : test %d, hash_alg %d, chain_order %d, " @@ -688,7 +706,7 @@ is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, (int) job->cipher_mode, (int) job->status); return 0; } -#endif + return 1; } @@ -714,7 +732,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) "jobs\n", __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_NULL_BURST) { @@ -723,7 +741,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); /* ======== test 2 : NULL jobs array */ @@ -733,7 +751,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) "jobs\n", __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_NULL_JOB) { @@ -742,7 +760,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); /* ========== test 3: invalid burst size */ @@ -753,7 +771,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) "jobs\n", __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_BURST_SIZE) { @@ -762,7 +780,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); } /* ======== test 4 : invalid job order */ @@ -773,8 +791,14 @@ test_burst_api(struct IMB_MGR *mb_mgr) /* fill in valid jobs */ for (i = 0; i < n_jobs; i++) { job = jobs[i]; +#ifndef __aarch64__ fill_in_job(job, IMB_CIPHER_CBC, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, IMB_ORDER_CIPHER_HASH, NULL, NULL); +#else /* __aarch64__ */ + fill_in_job(job, IMB_CIPHER_ZUC_EEA3, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, + IMB_ORDER_CIPHER_HASH, NULL, NULL); +#endif /* __aarch64__ */ + imb_set_session(mb_mgr, job); } /* set invalid job order */ @@ -786,7 +810,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) "jobs\n", __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_BURST_OOO) { @@ -795,7 +819,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); /* ======== test 5 : invalid job */ @@ -805,8 +829,14 @@ test_burst_api(struct IMB_MGR *mb_mgr) /* fill in valid jobs */ for (i = 0; i < n_jobs; i++) { job = jobs[i]; +#ifndef __aarch64__ fill_in_job(job, IMB_CIPHER_CBC, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, IMB_ORDER_CIPHER_HASH, NULL, NULL); +#else /* __aarch64__ */ + fill_in_job(job, IMB_CIPHER_ZUC_EEA3, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, + IMB_ORDER_CIPHER_HASH, NULL, NULL); +#endif /* __aarch64__ */ + imb_set_session(mb_mgr, job); } /* set a single invalid field */ @@ -819,7 +849,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_JOB_NULL_KEY) { @@ -827,7 +857,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_BURST, imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); /* check invalid job returned in jobs[0] */ if (jobs[0] != jobs[n_jobs - 1]) { @@ -837,7 +867,8 @@ test_burst_api(struct IMB_MGR *mb_mgr) return 1; } - printf("\n"); + if (!quiet_mode) + printf("\n"); if ((mb_mgr->features & IMB_FEATURE_SAFE_PARAM) == 0) return 0; @@ -852,7 +883,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_NULL_BURST) { @@ -861,7 +892,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) imb_get_strerror(err)); return 1; } - printf("."); + print_progress(); /* ======== test 7 : Invalid burst size */ @@ -872,7 +903,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_BURST_SIZE) { @@ -881,7 +912,9 @@ test_burst_api(struct IMB_MGR *mb_mgr) imb_get_strerror(err)); return 1; } - printf(".\n"); + print_progress(); + if (!quiet_mode) + printf("\n"); printf("FLUSH_BURST() API behavior test:\n"); @@ -891,7 +924,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) __func__, TEST_INVALID_BURST); return 1; } - printf("."); + print_progress(); err = imb_get_errno(mb_mgr); if (err != IMB_ERR_NULL_BURST) { @@ -900,7 +933,122 @@ test_burst_api(struct IMB_MGR *mb_mgr) imb_get_strerror(err)); return 1; } - printf(".\n"); + print_progress(); + + /* ======== test 8 : invalid suite_id */ + + while (IMB_GET_NEXT_BURST(mb_mgr, n_jobs, jobs) < n_jobs) + IMB_FLUSH_BURST(mb_mgr, n_jobs, jobs); + + /* fill in valid jobs */ + for (i = 0; i < n_jobs; i++) { + job = jobs[i]; +#ifndef __aarch64__ + fill_in_job(job, IMB_CIPHER_CBC, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, + IMB_ORDER_CIPHER_HASH, NULL, NULL); +#else /* __aarch64__ */ + fill_in_job(job, IMB_CIPHER_ZUC_EEA3, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, + IMB_ORDER_CIPHER_HASH, NULL, NULL); +#endif /* __aarch64__ */ + + if (i == (n_jobs - 1)) + memset(job->suite_id, 0, sizeof(job->suite_id)); /* bad suite_id */ + else + imb_set_session(mb_mgr, job); + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed " + "jobs\n", __func__, TEST_INVALID_BURST); + return 1; + } + print_progress(); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_BURST_SUITE_ID) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + print_progress(); + + /* ======== test 9 : session_d */ + + while (IMB_GET_NEXT_BURST(mb_mgr, n_jobs, jobs) < n_jobs) + IMB_FLUSH_BURST(mb_mgr, n_jobs, jobs); + + /* fill in valid jobs */ + for (i = 0; i < n_jobs; i++) { + job = jobs[i]; +#ifndef __aarch64__ + fill_in_job(job, IMB_CIPHER_CBC, IMB_DIR_ENCRYPT, IMB_AUTH_HMAC_SHA_256, + IMB_ORDER_CIPHER_HASH, NULL, NULL); +#else /* __aarch64__ */ + fill_in_job(job, IMB_CIPHER_ZUC_EEA3, IMB_DIR_ENCRYPT, IMB_AUTH_ZUC_EIA3_BITLEN, + IMB_ORDER_CIPHER_HASH, NULL, NULL); +#endif /* __aarch64__ */ + if (i > 0) { + /* + * Check if each call to session ID for the same cipher suite gives + * different ID. + */ + imb_set_session(mb_mgr, job); + if (job->session_id == jobs[i - 1]->session_id) { + printf("%s: test %d, unexpected/duplicate session_id value\n", + __func__, TEST_INVALID_BURST); + return 1; + } + } else { + /* NULL MB MGR pointer */ + imb_set_session(NULL, job); + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_MBMGR) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + print_progress(); + + /* NULL JOB pointer */ + imb_set_session(mb_mgr, NULL); + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_JOB) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + print_progress(); + + /* correct call at the end */ + imb_set_session(mb_mgr, job); + err = imb_get_errno(mb_mgr); + if (err != 0) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + } + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); + completed_jobs += IMB_FLUSH_BURST(mb_mgr, n_jobs, jobs); + if (completed_jobs != n_jobs) { + printf("%s: test %d, unexpected number of completed jobs\n", + __func__, TEST_INVALID_BURST); + return 1; + } + print_progress(); + + + /* ======== end */ + + if (!quiet_mode) + printf("\n"); return 0; } @@ -974,7 +1122,8 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) /* * Skip hash algorithms belonging to AEAD * algorithms, as the test is for authentication - * only algorithms */ + * only algorithms + */ if (check_aead(hash, cipher)) continue; @@ -987,12 +1136,13 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_SRC)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_AUTH_SRC_NULL, - IMB_ERR_JOB_NULL_SRC)) + &template_job, + TEST_AUTH_SRC_NULL, + IMB_ERR_JOB_NULL_SRC)) return 1; - printf("."); + print_progress(); } /* @@ -1027,16 +1177,17 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) template_job.sgl_state = IMB_SGL_COMPLETE; if (!is_submit_invalid(mb_mgr, &template_job, - TEST_AUTH_AUTH_TAG_OUTPUT_NULL, - IMB_ERR_JOB_NULL_AUTH)) + TEST_AUTH_AUTH_TAG_OUTPUT_NULL, + IMB_ERR_JOB_NULL_AUTH)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_AUTH_AUTH_TAG_OUTPUT_NULL, - IMB_ERR_JOB_NULL_AUTH)) + &template_job, + TEST_AUTH_AUTH_TAG_OUTPUT_NULL, + IMB_ERR_JOB_NULL_AUTH)) return 1; - printf("."); + print_progress(); } /* @@ -1071,15 +1222,16 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) template_job.sgl_state = IMB_SGL_COMPLETE; if (!is_submit_invalid(mb_mgr, &template_job, - TEST_AUTH_TAG_OUTPUT_LEN_ZERO, - IMB_ERR_JOB_AUTH_TAG_LEN)) + TEST_AUTH_TAG_OUTPUT_LEN_ZERO, + IMB_ERR_JOB_AUTH_TAG_LEN)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_AUTH_TAG_OUTPUT_LEN_ZERO, - IMB_ERR_JOB_AUTH_TAG_LEN)) + &template_job, + TEST_AUTH_TAG_OUTPUT_LEN_ZERO, + IMB_ERR_JOB_AUTH_TAG_LEN)) return 1; - printf("."); + print_progress(); } /* @@ -1160,16 +1312,17 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) break; } if (!is_submit_invalid(mb_mgr, &template_job, - TEST_AUTH_MSG_LEN_GT_MAX, - IMB_ERR_JOB_AUTH_LEN)) + TEST_AUTH_MSG_LEN_GT_MAX, + IMB_ERR_JOB_AUTH_LEN)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_AUTH_MSG_LEN_GT_MAX, - IMB_ERR_JOB_AUTH_LEN)) + &template_job, + TEST_AUTH_MSG_LEN_GT_MAX, + IMB_ERR_JOB_AUTH_LEN)) return 1; - printf("."); + print_progress(); } /* @@ -1188,10 +1341,10 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) #endif switch (hash) { - /* - * Cases below don't allow for zero length - * hash messages - */ + /* + * Cases below don't allow for zero length + * hash messages + */ case IMB_AUTH_HMAC_SHA_1: case IMB_AUTH_HMAC_SHA_224: case IMB_AUTH_HMAC_SHA_256: @@ -1225,12 +1378,13 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_AUTH_LEN)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_AUTH_MSG_LEN_ZERO, - IMB_ERR_JOB_AUTH_LEN)) + &template_job, + TEST_AUTH_MSG_LEN_ZERO, + IMB_ERR_JOB_AUTH_LEN)) return 1; - printf("."); + print_progress(); } /* @@ -1251,7 +1405,7 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) * for relevant algos */ switch (hash) { -#ifdef __x86_64__ +#ifndef __aarch64__ /* GMAC IVs must be not be 0 bytes */ case IMB_AUTH_AES_GMAC_128: case IMB_AUTH_AES_GMAC_192: @@ -1270,11 +1424,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_IV_LEN)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_IV_LEN, - IMB_ERR_JOB_IV_LEN)) + TEST_AUTH_IV_LEN, + IMB_ERR_JOB_IV_LEN)) return 1; - printf("."); + print_progress(); } /* @@ -1289,7 +1444,7 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) int skip = 1; switch (hash) { -#ifdef __x86_64__ +#ifndef __aarch64__ case IMB_AUTH_HMAC_SHA_1: case IMB_AUTH_HMAC_SHA_224: case IMB_AUTH_HMAC_SHA_256: @@ -1318,11 +1473,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) err_ipad)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_NULL_HMAC_IPAD, - err_ipad)) + TEST_AUTH_NULL_HMAC_IPAD, + err_ipad)) return 1; - printf("."); + print_progress(); fill_in_job(job, cipher, dir, hash, order, &chacha_ctx, @@ -1336,18 +1492,19 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) err_opad)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_NULL_HMAC_OPAD, - err_opad)) + TEST_AUTH_NULL_HMAC_OPAD, + err_opad)) return 1; - printf("."); + print_progress(); } +#ifndef __aarch64__ /* * Invalid XCBC key parameters */ -#ifdef __x86_64__ for (order = IMB_ORDER_CIPHER_HASH; order <= IMB_ORDER_HASH_CIPHER; order++) for (dir = IMB_DIR_ENCRYPT; dir <= IMB_DIR_DECRYPT; dir++) { @@ -1364,11 +1521,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_XCBC_K1_EXP)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_NULL_XCBC_K1_EXP, - IMB_ERR_JOB_NULL_XCBC_K1_EXP)) + TEST_AUTH_NULL_XCBC_K1_EXP, + IMB_ERR_JOB_NULL_XCBC_K1_EXP)) return 1; - printf("."); + print_progress(); fill_in_job(job, cipher, dir, hash, order, &chacha_ctx, @@ -1379,11 +1537,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_XCBC_K2)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, TEST_AUTH_NULL_XCBC_K2, IMB_ERR_JOB_NULL_XCBC_K2)) return 1; - printf("."); + print_progress(); fill_in_job(job, cipher, dir, hash, order, &chacha_ctx, @@ -1393,12 +1552,14 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_XCBC_K3, IMB_ERR_JOB_NULL_XCBC_K3)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, TEST_AUTH_NULL_XCBC_K3, IMB_ERR_JOB_NULL_XCBC_K3)) return 1; - printf("."); + print_progress(); } + /* * Invalid GHASH parameters */ @@ -1418,11 +1579,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_AUTH_KEY)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, TEST_AUTH_NULL_GHASH_KEY, IMB_ERR_JOB_NULL_AUTH_KEY)) return 1; - printf("."); + print_progress(); fill_in_job(job, cipher, dir, hash, order, &chacha_ctx, @@ -1433,11 +1595,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_GHASH_INIT_TAG)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_NULL_GHASH_INIT_TAG, - IMB_ERR_JOB_NULL_GHASH_INIT_TAG)) + TEST_AUTH_NULL_GHASH_INIT_TAG, + IMB_ERR_JOB_NULL_GHASH_INIT_TAG)) return 1; - printf("."); + print_progress(); } /* @@ -1456,15 +1619,16 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) job->u.GMAC._key = NULL; if (!is_submit_invalid(mb_mgr, job, - TEST_AUTH_NULL_GMAC_KEY, - IMB_ERR_JOB_NULL_AUTH_KEY)) + TEST_AUTH_NULL_GMAC_KEY, + IMB_ERR_JOB_NULL_AUTH_KEY)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_NULL_GMAC_KEY, - IMB_ERR_JOB_NULL_AUTH_KEY)) + TEST_AUTH_NULL_GMAC_KEY, + IMB_ERR_JOB_NULL_AUTH_KEY)) return 1; - printf("."); + print_progress(); fill_in_job(job, cipher, dir, hash, order, &chacha_ctx, @@ -1475,11 +1639,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_IV)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_NULL_GMAC_IV, - IMB_ERR_JOB_NULL_IV)) + TEST_AUTH_NULL_GMAC_IV, + IMB_ERR_JOB_NULL_IV)) return 1; - printf("."); + print_progress(); fill_in_job(job, cipher, dir, hash, order, &chacha_ctx, @@ -1490,19 +1655,22 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_IV_LEN)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_AUTH_GMAC_IV_LEN, - IMB_ERR_JOB_IV_LEN)) + TEST_AUTH_GMAC_IV_LEN, + IMB_ERR_JOB_IV_LEN)) return 1; - printf("."); + print_progress(); } } -#endif +#endif /* __aarch64__ */ + /* clean up */ while (IMB_FLUSH_JOB(mb_mgr) != NULL) ; - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -1558,12 +1726,13 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_SRC)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_SRC_NULL, - IMB_ERR_JOB_NULL_SRC)) + &template_job, + TEST_CIPH_SRC_NULL, + IMB_ERR_JOB_NULL_SRC)) return 1; - printf("."); + print_progress(); } /* @@ -1598,12 +1767,13 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_DST)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_DST_NULL, - IMB_ERR_JOB_NULL_DST)) + &template_job, + TEST_CIPH_DST_NULL, + IMB_ERR_JOB_NULL_DST)) return 1; - printf("."); + print_progress(); } /* @@ -1642,14 +1812,15 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_IV)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_IV_NULL, - IMB_ERR_JOB_NULL_IV)) + &template_job, + TEST_CIPH_IV_NULL, + IMB_ERR_JOB_NULL_IV)) return 1; - printf("."); + print_progress(); } - /* + /* * CIPHER_DIR = Invalid dir */ for (dir = 0; dir <= 10; dir++) { @@ -1668,6 +1839,7 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) cipher != IMB_CIPHER_ZUC_EEA3) continue; #endif + /* * Skip cipher algorithms belonging to AEAD * algorithms, as the test is for cipher @@ -1685,11 +1857,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_CIPH_DIR)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, &template_job, TEST_CIPH_DIR, IMB_ERR_JOB_CIPH_DIR)) return 1; - printf("."); + print_progress(); } } @@ -1729,14 +1902,15 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_KEY)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_ENC_KEY_NULL, - IMB_ERR_JOB_NULL_KEY)) + &template_job, + TEST_CIPH_ENC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) return 1; break; } - printf("."); + print_progress(); } /* ======== (decrypt test) @@ -1776,10 +1950,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_KEY)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_DEC_KEY_NULL, - IMB_ERR_JOB_NULL_KEY)) + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) return 1; break; case IMB_CIPHER_CNTR: @@ -1796,10 +1971,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_KEY)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_DEC_KEY_NULL, - IMB_ERR_JOB_NULL_KEY)) + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) return 1; break; case IMB_CIPHER_DOCSIS_SEC_BPI: @@ -1809,10 +1985,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_KEY)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_DEC_KEY_NULL, - IMB_ERR_JOB_NULL_KEY)) + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) return 1; template_job.enc_keys = template_job.dec_keys; @@ -1822,10 +1999,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_KEY)) return 1; + imb_set_session(mb_mgr, &template_job); if (!is_submit_burst_invalid(mb_mgr, - &template_job, - TEST_CIPH_DEC_KEY_NULL, - IMB_ERR_JOB_NULL_KEY)) + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) return 1; break; case IMB_CIPHER_NULL: @@ -1833,7 +2011,7 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) default: break; } - printf("."); + print_progress(); } /* @@ -1866,7 +2044,7 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) &chacha_ctx, &gcm_ctx); switch (cipher) { - /* skip ciphers that allow msg length 0 */ + /* skip ciphers that allow msg length 0 */ case IMB_CIPHER_GCM: case IMB_CIPHER_GCM_SGL: case IMB_CIPHER_CCM: @@ -1881,17 +2059,18 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) default: job->msg_len_to_cipher_in_bytes = 0; if (!is_submit_invalid(mb_mgr, job, - TEST_CIPH_MSG_LEN_ZERO, - IMB_ERR_JOB_CIPH_LEN)) + TEST_CIPH_MSG_LEN_ZERO, + IMB_ERR_JOB_CIPH_LEN)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, - job, - TEST_CIPH_MSG_LEN_ZERO, - IMB_ERR_JOB_CIPH_LEN)) + job, + TEST_CIPH_MSG_LEN_ZERO, + IMB_ERR_JOB_CIPH_LEN)) return 1; } - printf("."); + print_progress(); } /* @@ -1974,12 +2153,13 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_CIPH_LEN)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_CIPH_MSG_LEN_GT_MAX, - IMB_ERR_JOB_CIPH_LEN)) + TEST_CIPH_MSG_LEN_GT_MAX, + IMB_ERR_JOB_CIPH_LEN)) return 1; - printf("."); + print_progress(); } /* @@ -1989,7 +2169,7 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_CIPHER_MODE cipher_mode; uint64_t invalid_iv_len; } invalid_iv_lens[] = { -#if defined(__x86_64__) +#ifndef __aarch64__ /* IVs must be 16 bytes */ { IMB_CIPHER_CBC, 15 }, { IMB_CIPHER_CBC, 17 }, @@ -2001,8 +2181,10 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) { IMB_CIPHER_CNTR_BITLEN, 17 }, { IMB_CIPHER_PON_AES_CNTR, 15 }, { IMB_CIPHER_PON_AES_CNTR, 17 }, +#endif /* __aarch64__ */ { IMB_CIPHER_SNOW3G_UEA2_BITLEN, 15 }, { IMB_CIPHER_SNOW3G_UEA2_BITLEN, 17 }, +#ifndef __aarch64__ { IMB_CIPHER_SNOW_V_AEAD, 15 }, { IMB_CIPHER_SNOW_V_AEAD, 17 }, { IMB_CIPHER_SNOW_V, 15 }, @@ -2010,9 +2192,6 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) /* CCM IV must be 13 to 7 bytes */ { IMB_CIPHER_CCM, 6 }, { IMB_CIPHER_CCM, 14 }, - /* ECB IV must be 0 bytes */ - { IMB_CIPHER_ECB, 1 }, - { IMB_CIPHER_ECB, -1 }, /* CNTR IV must be 12 or 16 bytes */ { IMB_CIPHER_CNTR, 11 }, { IMB_CIPHER_CNTR, 14 }, @@ -2027,12 +2206,14 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) /* KASUMI IV must be 8 bytes */ { IMB_CIPHER_KASUMI_UEA1_BITLEN, 7 }, { IMB_CIPHER_KASUMI_UEA1_BITLEN, 9 }, +#endif /* __aarch64__ */ /* ZUC IV must be 16, 23 or 25 bytes */ { IMB_CIPHER_ZUC_EEA3, 15 }, { IMB_CIPHER_ZUC_EEA3, 17 }, { IMB_CIPHER_ZUC_EEA3, 22 }, { IMB_CIPHER_ZUC_EEA3, 24 }, { IMB_CIPHER_ZUC_EEA3, 26 }, +#ifndef __aarch64__ /* CHACHA20 IVs must be 12 bytes */ { IMB_CIPHER_CHACHA20, 15 }, { IMB_CIPHER_CHACHA20, 17 }, @@ -2043,17 +2224,7 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) /* GCM IVs must be not be 0 bytes */ { IMB_CIPHER_GCM, 0 }, { IMB_CIPHER_GCM_SGL, 0 }, -#elif defined(__aarch64__) - /* IVs must be 16 bytes */ - { IMB_CIPHER_SNOW3G_UEA2_BITLEN, 15 }, - { IMB_CIPHER_SNOW3G_UEA2_BITLEN, 17 }, - /* ZUC IV must be 16, 23 or 25 bytes */ - { IMB_CIPHER_ZUC_EEA3, 15 }, - { IMB_CIPHER_ZUC_EEA3, 17 }, - { IMB_CIPHER_ZUC_EEA3, 22 }, - { IMB_CIPHER_ZUC_EEA3, 24 }, - { IMB_CIPHER_ZUC_EEA3, 26 }, -#endif +#endif /* __aarch64__ */ }; dir = IMB_DIR_ENCRYPT; @@ -2085,13 +2256,15 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) /* skip some key lengths for specific ciphers */ switch (cipher) { - -#ifdef __x86_64__ +#ifndef __aarch64__ case IMB_CIPHER_CCM: case IMB_CIPHER_DOCSIS_SEC_BPI: +#endif /* __aarch64__ */ + case IMB_CIPHER_ZUC_EEA3: if (key_len == IMB_KEY_192_BYTES) continue; break; +#ifndef __aarch64__ case IMB_CIPHER_DES: case IMB_CIPHER_DOCSIS_DES: /* override default key len for DES */ @@ -2111,16 +2284,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) break; case IMB_CIPHER_CBCS_1_9: case IMB_CIPHER_PON_AES_CNTR: - case IMB_CIPHER_KASUMI_UEA1_BITLEN: - if (key_len != IMB_KEY_128_BYTES) - continue; - break; -#endif - case IMB_CIPHER_ZUC_EEA3: - if (key_len == IMB_KEY_192_BYTES) - continue; - break; +#endif /* __aarch64__ */ case IMB_CIPHER_SNOW3G_UEA2_BITLEN: +#ifndef __aarch64__ + case IMB_CIPHER_KASUMI_UEA1_BITLEN: +#endif /* __aarch64__ */ if (key_len != IMB_KEY_128_BYTES) continue; break; @@ -2133,11 +2301,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_IV_LEN)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_CIPH_IV_LEN, - IMB_ERR_JOB_IV_LEN)) + TEST_CIPH_IV_LEN, + IMB_ERR_JOB_IV_LEN)) return 1; - printf("."); + print_progress(); } } } @@ -2172,11 +2341,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_NULL_NEXT_IV)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, TEST_CIPH_NEXT_IV_NULL, IMB_ERR_JOB_NULL_NEXT_IV)) return 1; - printf("."); + print_progress(); } #endif @@ -2184,7 +2354,8 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) while (IMB_FLUSH_JOB(mb_mgr) != NULL) ; - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -2196,7 +2367,7 @@ test_job_invalid_misc_args(struct IMB_MGR *mb_mgr) { IMB_HASH_ALG hash; IMB_CIPHER_DIRECTION dir; - IMB_CIPHER_MODE cipher = IMB_CIPHER_NULL; + IMB_CIPHER_MODE cipher; IMB_CHAIN_ORDER order; struct IMB_JOB template_job; struct chacha20_poly1305_context_data chacha_ctx; @@ -2213,7 +2384,7 @@ test_job_invalid_misc_args(struct IMB_MGR *mb_mgr) */ for (order = IMB_ORDER_CIPHER_HASH; order <= IMB_ORDER_HASH_CIPHER; order++) -#ifdef __x86_64__ +#ifndef __aarch64__ for (dir = IMB_DIR_ENCRYPT; dir <= IMB_DIR_DECRYPT; dir++) { cipher = IMB_CIPHER_PON_AES_CNTR; hash = IMB_AUTH_PON_CRC_BIP; @@ -2234,14 +2405,17 @@ test_job_invalid_misc_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_PON_PLI)) return 1; + imb_set_session(mb_mgr, &template_job); + if (!is_submit_burst_invalid(mb_mgr, &template_job, TEST_INVALID_PON_PLI, IMB_ERR_JOB_PON_PLI)) return 1; - printf("."); + print_progress(); } - #endif + + /* * AEAD MSG_LEN > MAX */ @@ -2264,7 +2438,7 @@ test_job_invalid_misc_args(struct IMB_MGR *mb_mgr) switch (cipher) { /* skip algos with no max limit */ -#ifdef __x86_64__ +#ifndef __aarch64__ case IMB_CIPHER_PON_AES_CNTR: case IMB_CIPHER_SNOW_V_AEAD: case IMB_CIPHER_CHACHA20_POLY1305: @@ -2286,19 +2460,21 @@ test_job_invalid_misc_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_CIPH_LEN)) return 1; + imb_set_session(mb_mgr, job); if (!is_submit_burst_invalid(mb_mgr, job, - TEST_CIPH_MSG_LEN_GT_MAX, - IMB_ERR_JOB_CIPH_LEN)) + TEST_CIPH_MSG_LEN_GT_MAX, + IMB_ERR_JOB_CIPH_LEN)) return 1; - printf("."); + print_progress(); } /* clean up */ while (IMB_FLUSH_JOB(mb_mgr) != NULL) ; - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -2346,8 +2522,8 @@ submit_reset_check_job(struct IMB_MGR *mb_mgr, if (next_job->status != IMB_STATUS_COMPLETED) { printf("Returned job's status is not completed\n"); - printf("cipher = %u\n", cipher); - printf("imb errno = %u (%s)\n", + printf("cipher = %d\n", cipher); + printf("imb errno = %d (%s)\n", mb_mgr->imb_errno, imb_get_strerror(mb_mgr->imb_errno)); exit(0); @@ -2442,9 +2618,9 @@ test_reset_api(struct IMB_MGR *mb_mgr) } } -#ifdef __x86_64__ +#ifndef __aarch64__ /* Test AEAD algorithms */ - IMB_HASH_ALG aead_hash_algos[] = { + const IMB_HASH_ALG aead_hash_algos[] = { IMB_AUTH_AES_GMAC, IMB_AUTH_AES_CCM, IMB_AUTH_CHACHA20_POLY1305, @@ -2452,7 +2628,7 @@ test_reset_api(struct IMB_MGR *mb_mgr) IMB_AUTH_DOCSIS_CRC32, IMB_AUTH_SNOW_V_AEAD }; - IMB_CIPHER_MODE aead_cipher_algos[] = { + const IMB_CIPHER_MODE aead_cipher_algos[] = { IMB_CIPHER_GCM, IMB_CIPHER_CCM, IMB_CIPHER_CHACHA20_POLY1305, @@ -2490,12 +2666,14 @@ test_reset_api(struct IMB_MGR *mb_mgr) return 1; } -#endif +#endif /* __aarch64__ */ + /* clean up */ while (IMB_FLUSH_JOB(mb_mgr) != NULL) ; - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -2510,11 +2688,8 @@ api_test(struct IMB_MGR *mb_mgr) errors += test_job_api(mb_mgr); run++; - // To do... it's not implemented for arm platform -#ifdef __x86_64__ errors += test_burst_api(mb_mgr); run++; -#endif errors += test_job_invalid_mac_args(mb_mgr); run++; diff --git a/test/ccm_test.c b/test/kat-app/ccm_test.c similarity index 98% rename from test/ccm_test.c rename to test/kat-app/ccm_test.c index 3ce11f8ffb3f7afd1b1cc677046b42859024bc59..4f580984dbba45e8d7990c765ef284f82a44cc26 100644 --- a/test/ccm_test.c +++ b/test/kat-app/ccm_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -2444,21 +2444,24 @@ test_ccm_128_std_vectors(struct IMB_MGR *mb_mgr, const int vectors_cnt = sizeof(ccm_vectors) / sizeof(ccm_vectors[0]); int vect; - printf("AES-CCM-128 standard test vectors (N jobs = %d):\n", num_jobs); + if (!quiet_mode) + printf("AES-CCM-128 standard test vectors (N jobs = %d):\n", num_jobs); for (vect = 1; vect <= vectors_cnt; vect++) { const int idx = vect - 1; + if (!quiet_mode) { #ifdef DEBUG - printf("Standard vector [%d/%d] NONCELen:%d PktLen:%d " - "AADLen:%d AUTHlen:%d\n", - vect, vectors_cnt, - (int) ccm_vectors[idx].nonce_len, - (int) ccm_vectors[idx].packet_len, - (int) ccm_vectors[idx].clear_len, - (int) ccm_vectors[idx].auth_len); + printf("Standard vector [%d/%d] NONCELen:%d PktLen:%d " + "AADLen:%d AUTHlen:%d\n", + vect, vectors_cnt, + (int) ccm_vectors[idx].nonce_len, + (int) ccm_vectors[idx].packet_len, + (int) ccm_vectors[idx].clear_len, + (int) ccm_vectors[idx].auth_len); #else - printf("."); + printf("."); #endif + } if (test_ccm(mb_mgr, &ccm_vectors[idx], IMB_DIR_ENCRYPT, 1, num_jobs, 16)) { @@ -2492,7 +2495,8 @@ test_ccm_128_std_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } static void @@ -2503,20 +2507,25 @@ test_ccm_256_std_vectors(struct IMB_MGR *mb_mgr, const int vectors_cnt = DIM(ccm_256_vectors); int vect; - printf("AES-CCM-256 standard test vectors (N jobs = %d):\n", num_jobs); + if (!quiet_mode) + printf("AES-CCM-256 standard test vectors (N jobs = %d):\n", + num_jobs); for (vect = 1; vect <= vectors_cnt; vect++) { const int idx = vect - 1; + + if (!quiet_mode) { #ifdef DEBUG - printf("Standard vector [%d/%d] NONCELen:%d PktLen:%d " - "AADLen:%d AUTHlen:%d\n", - vect, vectors_cnt, - (int) ccm_256_vectors[idx].nonce_len, - (int) ccm_256_vectors[idx].packet_len, - (int) ccm_256_vectors[idx].clear_len, - (int) ccm_256_vectors[idx].auth_len); + printf("Standard vector [%d/%d] NONCELen:%d PktLen:%d " + "AADLen:%d AUTHlen:%d\n", + vect, vectors_cnt, + (int) ccm_256_vectors[idx].nonce_len, + (int) ccm_256_vectors[idx].packet_len, + (int) ccm_256_vectors[idx].clear_len, + (int) ccm_256_vectors[idx].auth_len); #else - printf("."); + printf("."); #endif + } if (test_ccm(mb_mgr, &ccm_256_vectors[idx], IMB_DIR_ENCRYPT, 1, num_jobs, 32)) { @@ -2550,7 +2559,8 @@ test_ccm_256_std_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } diff --git a/test/chacha20_poly1305_test.c b/test/kat-app/chacha20_poly1305_test.c similarity index 98% rename from test/chacha20_poly1305_test.c rename to test/kat-app/chacha20_poly1305_test.c index 60cf960b31cfbe4a26a0c95d1b7ac2ac1a18ed04..d89afad376f20c371738a5b1190de9f984cf2423 100644 --- a/test/chacha20_poly1305_test.c +++ b/test/kat-app/chacha20_poly1305_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -502,6 +502,8 @@ test_aead(struct IMB_MGR *mb_mgr, job->auth_tag_output_len_in_bytes = 16; job->user_data = auths[i]; + + imb_set_session(mb_mgr, job); } uint32_t completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); @@ -586,15 +588,18 @@ test_aead_vectors(struct IMB_MGR *mb_mgr, { size_t vect; - printf("%s (N jobs = %d):\n", banner, num_jobs); + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); for (vect = 0; vect < vec_array_size; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("Vector [%d/%d], M len: %d\n", - (int) vect + 1, (int) vec_array_size, - (int) vec_array[vect].msg_len); + printf("Vector [%d/%d], M len: %d\n", + (int) vect + 1, (int) vec_array_size, + (int) vec_array[vect].msg_len); #else - printf("."); + printf("."); #endif + } if (test_aead(mb_mgr, &vec_array[vect], IMB_DIR_ENCRYPT, num_jobs, 1)) { @@ -632,7 +637,8 @@ test_aead_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } diff --git a/test/chacha_test.c b/test/kat-app/chacha_test.c similarity index 97% rename from test/chacha_test.c rename to test/kat-app/chacha_test.c index 6532756a7878ccbfeda8bf10b79daa2721fbfeb3..36b709fa978248136446730c22b3ce9172fdd97e 100644 --- a/test/chacha_test.c +++ b/test/kat-app/chacha_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -357,15 +357,19 @@ test_chacha_vectors(struct IMB_MGR *mb_mgr, DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); - printf("%s (N jobs = %d):\n", banner, num_jobs); + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); for (vect = 0; vect < vec_cnt; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("[%d/%d] Standard vector key_len:%d\n", - vect + 1, vec_cnt, - (int) vec_tab[vect].Klen); + printf("[%d/%d] Standard vector key_len:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].Klen); #else - printf("."); + printf("."); #endif + } + memcpy(enc_keys, vec_tab[vect].K, vec_tab[vect].Klen); memcpy(dec_keys, vec_tab[vect].K, vec_tab[vect].Klen); @@ -421,7 +425,8 @@ test_chacha_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } int diff --git a/test/chained_test.c b/test/kat-app/chained_test.c similarity index 89% rename from test/chained_test.c rename to test/kat-app/chained_test.c index 0da2aebafe2dfd8018767f5a4ccdc68ee91c22ce..7ac6375da085038497370e3918f3e09c77c47493 100644 --- a/test/chained_test.c +++ b/test/kat-app/chained_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -380,39 +380,28 @@ test_chained_vectors(struct IMB_MGR *mb_mgr, const int vec_cnt, const struct chained_vector *vec_tab, const char *banner, const IMB_CIPHER_MODE cipher, - const IMB_HASH_ALG hash, - unsigned hash_block_size, int num_jobs) + const IMB_HASH_ALG hash, int num_jobs) { int vect; DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); - uint8_t *buf = NULL; - uint8_t *hash_key = NULL; DECLARE_ALIGNED(uint8_t ipad_hash[128], 16); DECLARE_ALIGNED(uint8_t opad_hash[128], 16); - unsigned hash_key_len, i; - - buf = malloc(hash_block_size); - if (buf == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto exit; - } - - hash_key = malloc(hash_block_size); - if (hash_key == NULL) { - fprintf(stderr, "Can't allocate key memory\n"); - goto exit; - } + unsigned i; - printf("%s (N jobs = %d):\n", banner, num_jobs); + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); for (vect = 0; vect < vec_cnt; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("[%d/%d] Standard vector key_len:%d\n", - vect + 1, vec_cnt, - (int) vec_tab[vect].cipher_key_len); + printf("[%d/%d] Standard vector key_len:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].cipher_key_len); #else - printf("."); + printf("."); #endif + } + /* prepare the cipher key */ switch (vec_tab[vect].cipher_key_len) { case 16: @@ -430,29 +419,10 @@ test_chained_vectors(struct IMB_MGR *mb_mgr, break; } - /* prepare the hash key */ - memset(hash_key, 0, hash_block_size); - if (vec_tab[vect].hash_key_len <= hash_block_size) { - memcpy(hash_key, vec_tab[vect].hash_key, - vec_tab[vect].hash_key_len); - hash_key_len = (int) vec_tab[vect].hash_key_len; - } else { - IMB_SHA1(mb_mgr, vec_tab[vect].hash_key, - vec_tab[vect].hash_key_len, hash_key); - hash_key_len = hash_block_size; - } - - /* compute ipad hash */ - memset(buf, 0x36, hash_block_size); - for (i = 0; i < hash_key_len; i++) - buf[i] ^= hash_key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); - - /* compute opad hash */ - memset(buf, 0x5c, hash_block_size); - for (i = 0; i < hash_key_len; i++) - buf[i] ^= hash_key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_1, + vec_tab[vect].cipher_key, + vec_tab[vect].cipher_key_len, + ipad_hash, opad_hash); for (i = 0; i < DIM(test_sets); i++) { unsigned in_place; @@ -477,11 +447,8 @@ test_chained_vectors(struct IMB_MGR *mb_mgr, } } } - printf("\n"); - -exit: - free(buf); - free(hash_key); + if (!quiet_mode) + printf("\n"); } int @@ -501,7 +468,7 @@ chained_test(struct IMB_MGR *mb_mgr) chained_vectors, "AES-CBC + SHA1-HMAC standard test vectors", IMB_CIPHER_CBC, IMB_AUTH_HMAC_SHA_1, - IMB_SHA1_BLOCK_SIZE, num_jobs_tab[i]); + num_jobs_tab[i]); errors += test_suite_end(&ctx); diff --git a/test/clear_mem_test.c b/test/kat-app/clear_mem_test.c similarity index 96% rename from test/clear_mem_test.c rename to test/kat-app/clear_mem_test.c index aaca00e5e9b601e7be44ff711658d29a3ac4d6dc..a269f7525d37b84e8ebb6894aa07bdf8ae3f1b2c 100644 --- a/test/clear_mem_test.c +++ b/test/kat-app/clear_mem_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -31,10 +31,7 @@ #include #include "ipsec-mb.h" - -#ifdef __x86_64__ #include "gcm_ctr_vectors_test.h" -#endif /* __x86_64__ */ #include "utils.h" @@ -143,9 +140,11 @@ int clear_mem_test(struct IMB_MGR *mb_mgr) free(buf); - printf("."); + if (!quiet_mode) + printf("."); } - printf("\n"); + if (!quiet_mode) + printf("\n"); errors = test_suite_end(&ctx); diff --git a/test/kat-app/cmac_test.c b/test/kat-app/cmac_test.c new file mode 100644 index 0000000000000000000000000000000000000000..47aee4ff684b0072b578efc62cad51db9517aafa --- /dev/null +++ b/test/kat-app/cmac_test.c @@ -0,0 +1,671 @@ +/***************************************************************************** + Copyright (c) 2018-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "gcm_ctr_vectors_test.h" +#include "utils.h" +#include "mac_test.h" + +enum cmac_type { + CMAC_128 = 0, + CMAC_128_BITLEN, + CMAC_256, +}; + +int cmac_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test cmac_128_test_json[]; +extern const struct mac_test cmac_256_test_json[]; +extern const struct mac_test cmac_3gpp_test_json[]; + +static const struct cmac_subkeys { + const char *key; + const char *sub_key1; + const char *sub_key2; +} cmac_128_subkeys[] = { + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\xfb\xee\xd6\x18\x35\x71\x33\x66\x7c\x85\xe0\x8f\x72\x36\xa8\xde", + "\xf7\xdd\xac\x30\x6a\xe2\x66\xcc\xf9\x0b\xc1\x1e\xe4\x6d\x51\x3b" + }, + { NULL, NULL, NULL } +}; + +static const struct cmac_subkeys cmac_256_subkeys[] = { + { + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\xca\xd1\xed\x03\x29\x9e\xed\xac\x2e\x9a\x99\x80\x86\x21\x50\x2f", + "\x95\xa3\xda\x06\x53\x3d\xdb\x58\x5d\x35\x33\x01\x0c\x42\xa0\xd9" + }, + { + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\xca\xd1\xed\x03\x29\x9e\xed\xac\x2e\x9a\x99\x80\x86\x21\x50\x2f", + "\x95\xa3\xda\x06\x53\x3d\xdb\x58\x5d\x35\x33\x01\x0c\x42\xa0\xd9" + }, + { + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\xca\xd1\xed\x03\x29\x9e\xed\xac\x2e\x9a\x99\x80\x86\x21\x50\x2f", + "\x95\xa3\xda\x06\x53\x3d\xdb\x58\x5d\x35\x33\x01\x0c\x42\xa0\xd9" + }, + { + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\xca\xd1\xed\x03\x29\x9e\xed\xac\x2e\x9a\x99\x80\x86\x21\x50\x2f", + "\x95\xa3\xda\x06\x53\x3d\xdb\x58\x5d\x35\x33\x01\x0c\x42\xa0\xd9" + }, + {NULL, NULL, NULL} +}; + +static const struct cmac_subkeys cmac_3gpp_subkeys[] = { + { + "\x2b\xd6\x45\x9f\x82\xc5\xb3\x00\x95\x2c\x49\x10\x48\x81\xff\x48", + "\xdc\x84\xc2\x70\xb5\xbf\x83\xf9\x6f\x90\xbe\x18\x8d\x3f\x64\x18", + "\xb9\x09\x84\xe1\x6b\x7f\x07\xf2\xdf\x21\x7c\x31\x1a\x7e\xc8\xb7" + }, + { + "\xd3\xc5\xd5\x92\x32\x7f\xb1\x1c\x40\x35\xc6\x68\x0a\xf8\xc6\xd1", + "\x36\xe3\xe5\x32\x26\x52\x2b\xa6\xc0\xa4\x23\x6b\xcb\xbf\x0c\xe3", + "\x6d\xc7\xca\x64\x4c\xa4\x57\x4d\x81\x48\x46\xd7\x97\x7e\x19\xc6" + }, + { + "\x7e\x5e\x94\x43\x1e\x11\xd7\x38\x28\xd7\x39\xcc\x6c\xed\x45\x73", + "\xaf\x16\x8c\x50\x6a\xf0\x3c\xf3\xa4\x4a\xbf\x1a\x61\x34\xc1\x59", + "\x5e\x2d\x18\xa0\xd5\xe0\x79\xe7\x48\x95\x7e\x34\xc2\x69\x82\x35" + }, + { + "\xd3\x41\x9b\xe8\x21\x08\x7a\xcd\x02\x12\x3a\x92\x48\x03\x33\x59", + "\x0a\x9b\xa0\x10\x5b\x3d\x9a\x43\x47\xe6\x56\x15\x4e\x6d\x37\xc8", + "\x15\x37\x40\x20\xb6\x7b\x34\x86\x8f\xcc\xac\x2a\x9c\xda\x6f\x90" + }, + { + "\x83\xfd\x23\xa2\x44\xa7\x4c\xf3\x58\xda\x30\x19\xf1\x72\x26\x35", + "\x3b\xec\x38\xae\x79\x0d\x59\x58\xe0\x9b\x73\xab\x61\xbd\x48\x0f", + "\x77\xd8\x71\x5c\xf2\x1a\xb2\xb1\xc1\x36\xe7\x56\xc3\x7a\x90\x1e" + }, + { + "\x68\x32\xa6\x5c\xff\x44\x73\x62\x1e\xbd\xd4\xba\x26\xa9\x21\xfe", + "\xca\x02\x47\x87\x0f\xc2\x7f\xad\x1b\x17\xe1\xa1\x48\xb0\x2d\x8d", + "\x94\x04\x8f\x0e\x1f\x84\xff\x5a\x36\x2f\xc3\x42\x91\x60\x5b\x9d" + }, + { + "\x5d\x0a\x80\xd8\x13\x4a\xe1\x96\x77\x82\x4b\x67\x1e\x83\x8a\xf4", + "\x30\x65\xc4\x53\xf7\x72\x72\xe1\x79\xef\x65\x04\x7d\xc9\xfc\x3d", + "\x60\xcb\x88\xa7\xee\xe4\xe5\xc2\xf3\xde\xca\x08\xfb\x93\xf8\x7a" + }, + { + "\xb3\x12\x0f\xfd\xb2\xcf\x6a\xf4\xe7\x3e\xaf\x2e\xf4\xeb\xec\x69", + "\x58\xc8\xbb\x9a\xe4\x22\x92\xc3\xb1\x73\x90\xc8\xf5\x58\x58\xb6", + "\xb1\x91\x77\x35\xc8\x45\x25\x87\x62\xe7\x21\x91\xea\xb0\xb1\x6c" + }, + {NULL, NULL, NULL} +}; + +static int +cmac_subkey_test(const struct cmac_subkeys *skeys, + uint32_t *skey1, uint32_t *skey2) +{ + uint32_t sub_key_size = sizeof(skey1); + + if (memcmp(skeys->sub_key1, skey1, sub_key_size)) { + printf("sub-key1 mismatched\n"); + hexdump(stderr, "Received", &skey1, sub_key_size); + hexdump(stderr, "Expected", (const void *) skeys->sub_key1, sub_key_size); + return 0; + } + + sub_key_size = sizeof(skey2); + + if (memcmp(skeys->sub_key2, skey2, sub_key_size)) { + printf("sub-key2 mismatched\n"); + hexdump(stderr, "Received", &skey2, sub_key_size); + hexdump(stderr, "Expected", (const void *) skeys->sub_key2, sub_key_size); + return 0; + } + return 1; +} + +static int +cmac_job_ok(const struct mac_test *vec, + const struct IMB_JOB *job, + const uint8_t *auth, + const uint8_t *padding, + const size_t sizeof_padding) +{ + const size_t auth_len = job->auth_tag_output_len_in_bytes; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("%d Error status:%d", __LINE__, job->status); + return 0; + } + + /* hash checks */ + if (memcmp(padding, &auth[sizeof_padding + auth_len], + sizeof_padding)) { + printf("hash overwrite tail\n"); + hexdump(stderr, "Target", + &auth[sizeof_padding + auth_len], sizeof_padding); + return 0; + } + + if (memcmp(padding, &auth[0], sizeof_padding)) { + printf("hash overwrite head\n"); + hexdump(stderr, "Target", &auth[0], sizeof_padding); + return 0; + } + + if (memcmp(vec->tag, &auth[sizeof_padding], auth_len)) { + printf("hash mismatched\n"); + hexdump(stderr, "Received", &auth[sizeof_padding], + auth_len); + hexdump(stderr, "Expected", vec->tag, + auth_len); + return 0; + } + return 1; +} + +static int +test_cmac(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const struct cmac_subkeys *subKeys, + const int dir, + const int num_jobs, + const enum cmac_type type) +{ + DECLARE_ALIGNED(uint32_t expkey[4*15], 16); + DECLARE_ALIGNED(uint32_t dust[4*15], 16); + uint32_t skey1[4], skey2[4]; + struct IMB_JOB *job; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + int i = 0, jobs_rx = 0, ret = -1; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + auths[i] = malloc(16 + (sizeof(padding) * 2)); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + + memset(auths[i], -1, 16 + (sizeof(padding) * 2)); + } + + if ((type == CMAC_128) || (type == CMAC_128_BITLEN)) { + IMB_AES_KEYEXP_128(mb_mgr, vec->key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, expkey, skey1, skey2); + } else { /* AES-CMAC-256 */ + IMB_AES_KEYEXP_256(mb_mgr, vec->key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_256(mb_mgr, expkey, skey1, skey2); + } + + if (!cmac_subkey_test(subKeys, skey1, skey2)) + goto end; + + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + /** + * Submit all jobs then flush any outstanding jobs + */ + for (i = 0; i < num_jobs; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + + switch (type) { + case CMAC_128: + job->hash_alg = IMB_AUTH_AES_CMAC; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + break; + case CMAC_128_BITLEN: + job->hash_alg = IMB_AUTH_AES_CMAC_BITLEN; + /* check for std or 3gpp vectors + scale len if necessary */ + job->msg_len_to_hash_in_bits = vec->msgSize; + break; + case CMAC_256: + job->hash_alg = IMB_AUTH_AES_CMAC_256; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + break; + default: + printf("Invalid CMAC type specified\n"); + goto end; + } + job->u.CMAC._key_expanded = expkey; + job->u.CMAC._skey1 = skey1; + job->u.CMAC._skey2 = skey2; + job->src = (const void *) vec->msg; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize / 8; + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job) { + jobs_rx++; + if (num_jobs < 4) { + printf("%d Unexpected return from submit_job\n", + __LINE__); + goto end; + } + if (!cmac_job_ok(vec, job, job->user_data, padding, + sizeof(padding))) + goto end; + } + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + jobs_rx++; + + if (!cmac_job_ok(vec, job, job->user_data, padding, + sizeof(padding))) + goto end; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + + /** + * Submit each job and flush immediately + */ + for (i = 0; i < num_jobs; i++) { + struct IMB_JOB *first_job = NULL; + + job = IMB_GET_NEXT_JOB(mb_mgr); + first_job = job; + + job->cipher_direction = dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + + switch (type) { + case CMAC_128: + job->hash_alg = IMB_AUTH_AES_CMAC; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + break; + case CMAC_128_BITLEN: + job->hash_alg = IMB_AUTH_AES_CMAC_BITLEN; + /* check for std or 3gpp vectors + scale len if necessary */ + job->msg_len_to_hash_in_bits = vec->msgSize; + break; + case CMAC_256: + job->hash_alg = IMB_AUTH_AES_CMAC_256; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + break; + default: + printf("Invalid CMAC type specified\n"); + goto end; + } + job->u.CMAC._key_expanded = expkey; + job->u.CMAC._skey1 = skey1; + job->u.CMAC._skey2 = skey2; + job->src = (const void *) vec->msg; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize / 8; + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job != NULL) { + printf("Received job, expected NULL\n"); + goto end; + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + if (job != first_job) { + printf("Invalid return job received\n"); + goto end; + } + if (!cmac_job_ok(vec, job, job->user_data, padding, + sizeof(padding))) + goto end; + } + } + + ret = 0; + + end: + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static void +test_cmac_std_vectors(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx, + const int num_jobs) +{ + const struct mac_test *v = cmac_128_test_json; + const struct cmac_subkeys *sk = cmac_128_subkeys; + + if (!quiet_mode) + printf("AES-CMAC-128 standard test vectors (N jobs = %d):\n", + num_jobs); + for (; v->msg != NULL; v++, sk++) { + if (!quiet_mode) { +#ifdef DEBUG + printf("Standard CMAC-128 vector %zu Message length: %zu, " + "Tag length:%zu\n", + v->tcId, + v->msgSize / 8, + v->tagSize / 8); +#else + printf("."); +#endif + } + + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_ENCRYPT, num_jobs, CMAC_128)) { + printf("error #%zu encrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_DECRYPT, num_jobs, CMAC_128)) { + printf("error #%zu decrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + } + if (!quiet_mode) + printf("\n"); +} + +static void +test_cmac_256_std_vectors(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx, + const int num_jobs) +{ + const struct mac_test *v = cmac_256_test_json; + const struct cmac_subkeys *sk = cmac_256_subkeys; + + if (!quiet_mode) + printf("AES-CMAC-256 standard test vectors (N jobs = %d):\n", + num_jobs); + for (; v->msg != NULL; v++, sk++) { + if (!quiet_mode) { +#ifdef DEBUG + printf("Standard CMAC-256 vector %zu Message length: %zu, " + "Tag length:%zu\n", + v->tcId, + v->msgSize / 8, + v->tagSize / 8); +#else + printf("."); +#endif + } + + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_ENCRYPT, num_jobs, CMAC_256)) { + printf("error #%zu encrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_DECRYPT, num_jobs, CMAC_256)) { + printf("error #%zu decrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + } + if (!quiet_mode) + printf("\n"); +} + +static void +test_cmac_bitlen_std_vectors(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx, + const int num_jobs) +{ + const struct mac_test *v = cmac_128_test_json; + const struct cmac_subkeys *sk = cmac_128_subkeys; + + if (!quiet_mode) + printf("AES-CMAC-128 BITLEN standard test vectors " + "(N jobs = %d):\n", num_jobs); + for (; v->msg != NULL; v++, sk++) { + if (!quiet_mode) { +#ifdef DEBUG + printf("Standard bit length vector %zu Message length (bits): %zu, " + "Tag length:%zu\n", + v->tcId, + v->msgSize, + v->tagSize / 8); +#else + printf("."); +#endif + } + + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_ENCRYPT, num_jobs, CMAC_128_BITLEN)) { + printf("error #%zu encrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_DECRYPT, num_jobs, CMAC_128_BITLEN)) { + printf("error #%zu decrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + } + if (!quiet_mode) + printf("\n"); +} + +static void +test_cmac_bitlen_3gpp_vectors(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx, + const int num_jobs) +{ + const struct mac_test *v = cmac_3gpp_test_json; + const struct cmac_subkeys *sk = cmac_3gpp_subkeys; + + if (!quiet_mode) + printf("AES-CMAC-128 BITLEN 3GPP test vectors (N jobs = %d):\n", + num_jobs); + for (; v->msg != NULL; v++, sk++) { + if (!quiet_mode) { +#ifdef DEBUG + printf("3gpp vector %zu Message length (bits): %zu, " + "Tag length:%zu\n", + v->tcId, + v->msgSize, + v->tagSize / 8); +#else + printf("."); +#endif + } + + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_ENCRYPT, num_jobs, CMAC_128_BITLEN)) { + printf("error #%zu encrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_cmac(mb_mgr, v, sk, + IMB_DIR_DECRYPT, num_jobs, CMAC_128_BITLEN)) { + printf("error #%zu decrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + } + if (!quiet_mode) + printf("\n"); +} + +int +cmac_test(struct IMB_MGR *mb_mgr) +{ + int i, errors = 0; + struct test_suite_context ctx; + + /* CMAC 128 with standard vectors */ + test_suite_start(&ctx, "AES-CMAC-128"); + for (i = 1; i < 20; i++) + test_cmac_std_vectors(mb_mgr, &ctx, i); + errors += test_suite_end(&ctx); + + /* CMAC 128 BITLEN with standard vectors */ + test_suite_start(&ctx, "AES-CMAC-128-BIT-LENGTH"); + for (i = 1; i < 20; i++) + test_cmac_bitlen_std_vectors(mb_mgr, &ctx, i); + + /* CMAC 128 BITLEN with 3GPP vectors */ + for (i = 1; i < 20; i++) + test_cmac_bitlen_3gpp_vectors(mb_mgr, &ctx, i); + errors += test_suite_end(&ctx); + + /* CMAC 256 with standard vectors */ + test_suite_start(&ctx, "AES-CMAC-256"); + for (i = 1; i < 20; i++) + test_cmac_256_std_vectors(mb_mgr, &ctx, i); + errors += test_suite_end(&ctx); + + return errors; +} diff --git a/test/kat-app/cmac_test.json.c b/test/kat-app/cmac_test.json.c new file mode 100644 index 0000000000000000000000000000000000000000..e0b8abbab6dc98bd4a9b94f9a4bc2f0eb6f24375 --- /dev/null +++ b/test/kat-app/cmac_test.json.c @@ -0,0 +1,393 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* CMAC, RFC4493 */ +#include "mac_test.h" + +const struct mac_test cmac_128_test_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc4493 */ + {128, 128, 1, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xbb\x1d\x69\x29\xe9\x59\x37\x28\x7f\xa3\x7d\x12\x9b\x75\x67\x46", 1, 0, + NULL, 0}, + {128, 128, 2, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x07\x0a\x16\xb4\x6b\x4d\x41\x44\xf7\x9b\xdd\x9d\xd0\x4a\x28\x7c", 1, 128, + NULL, 0}, + {128, 128, 3, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xdf\xa6\x67\x47\xde\x9a\xe6\x30\x30\xca\x32\x61\x14\x97\xc8\x27", 1, 320, + NULL, 0}, + {128, 128, 4, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x51\xf0\xbe\xbf\x7e\x3b\x9d\x92\xfc\x49\x74\x17\x79\x36\x3c\xfe", 1, 512, + NULL, 0}, + {128, 120, 5, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xbb\x1d\x69\x29\xe9\x59\x37\x28\x7f\xa3\x7d\x12\x9b\x75\x67\x46", 1, 0, + NULL, 0}, + {128, 120, 6, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x07\x0a\x16\xb4\x6b\x4d\x41\x44\xf7\x9b\xdd\x9d\xd0\x4a\x28\x7c", 1, 128, + NULL, 0}, + {128, 120, 7, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xdf\xa6\x67\x47\xde\x9a\xe6\x30\x30\xca\x32\x61\x14\x97\xc8\x27", 1, 320, + NULL, 0}, + {128, 120, 8, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x51\xf0\xbe\xbf\x7e\x3b\x9d\x92\xfc\x49\x74\x17\x79\x36\x3c\xfe", 1, 512, + NULL, 0}, + {128, 96, 9, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xbb\x1d\x69\x29\xe9\x59\x37\x28\x7f\xa3\x7d\x12\x9b\x75\x67\x46", 1, 0, + NULL, 0}, + {128, 96, 10, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x07\x0a\x16\xb4\x6b\x4d\x41\x44\xf7\x9b\xdd\x9d\xd0\x4a\x28\x7c", 1, 128, + NULL, 0}, + {128, 96, 11, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xdf\xa6\x67\x47\xde\x9a\xe6\x30\x30\xca\x32\x61\x14\x97\xc8\x27", 1, 320, + NULL, 0}, + {128, 96, 12, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x51\xf0\xbe\xbf\x7e\x3b\x9d\x92\xfc\x49\x74\x17\x79\x36\x3c\xfe", 1, 512, + NULL, 0}, + {128, 32, 13, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xbb\x1d\x69\x29\xe9\x59\x37\x28\x7f\xa3\x7d\x12\x9b\x75\x67\x46", 1, 0, + NULL, 0}, + {128, 32, 14, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x07\x0a\x16\xb4\x6b\x4d\x41\x44\xf7\x9b\xdd\x9d\xd0\x4a\x28\x7c", 1, 128, + NULL, 0}, + {128, 32, 15, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xdf\xa6\x67\x47\xde\x9a\xe6\x30\x30\xca\x32\x61\x14\x97\xc8\x27", 1, 320, + NULL, 0}, + {128, 32, 16, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x51\xf0\xbe\xbf\x7e\x3b\x9d\x92\xfc\x49\x74\x17\x79\x36\x3c\xfe", 1, 512, + NULL, 0}, + {128, 128, 17, + "\x2b\x7e\x15\x16\x28\xae\xd2\xa6\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xdc\x87\xcd\xcf\x77\xa2\xf1\x82\x9e\x01\x2c\x4d\x31\xaf\x2f\x8b", 1, 64, + NULL, 0}, + {0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0} + }; + +const struct mac_test cmac_256_test_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc4493 */ + {256, 128, 1, + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35" + "\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x02\x89\x62\xf6\x1b\x7b\xf8\x9e\xfc\x6b\x55\x1f\x46\x67\xd9\x83", 1, 0, + NULL, 0}, + {256, 128, 2, + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35" + "\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x28\xa7\x02\x3f\x45\x2e\x8f\x82\xbd\x4b\xf2\x8d\x8c\x37\xc3\x5c", 1, 128, + NULL, 0}, + {256, 128, 3, + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35" + "\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\x15\x67\x27\xdc\x08\x78\x94\x4a\x02\x3c\x1f\xe0\x3b\xad\x6d\x93", 1, 160, + NULL, 0}, + {256, 128, 4, + "\x60\x3d\xeb\x10\x15\xca\x71\xbe\x2b\x73\xae\xf0\x85\x7d\x77\x81\x1f\x35" + "\x2c\x07\x3b\x61\x08\xd7\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96\xe9\x3d\x7e\x11\x73\x93\x17\x2a\xae\x2d" + "\x8a\x57\x1e\x03\xac\x9c\x9e\xb7\x6f\xac\x45\xaf\x8e\x51\x30\xc8\x1c\x46" + "\xa3\x5c\xe4\x11\xe5\xfb\xc1\x19\x1a\x0a\x52\xef\xf6\x9f\x24\x45\xdf\x4f" + "\x9b\x17\xad\x2b\x41\x7b\xe6\x6c\x37\x10", + "\xe1\x99\x21\x90\x54\x9f\x6e\xd5\x69\x6a\x2c\x05\x6c\x31\x54\x10", 1, 512, + NULL, 0}, + {0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0} + }; + +const struct mac_test cmac_3gpp_test_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc4493 */ + {128, 32, 1, + "\x2b\xd6\x45\x9f\x82\xc5\xb3\x00\x95\x2c\x49\x10\x48\x81\xff\x48", + "\x38\xa6\xf0\x56\xc0\x00\x00\x00\x33\x32\x34\x62\x63\x39\x38\x40", + "\x11\x8c\x6e\xb8", 1, 122, NULL, 0}, + {128, 32, 2, + "\xd3\xc5\xd5\x92\x32\x7f\xb1\x1c\x40\x35\xc6\x68\x0a\xf8\xc6\xd1", + "\x39\x8a\x59\xb4\xd4\x00\x00\x00\x48\x45\x83\xd5\xaf\xe0\x82\xae", + "\xb9\x37\x87\xe6", 1, 128, NULL, 0}, + {128, 32, 3, + "\x7e\x5e\x94\x43\x1e\x11\xd7\x38\x28\xd7\x39\xcc\x6c\xed\x45\x73", + "\x36\xaf\x61\x44\xc4\x00\x00\x00\xb3\xd3\xc9\x17\x0a\x4e\x16\x32\xf6\x0f" + "\x86\x10\x13\xd2\x2d\x84\xb7\x26\xb6\xa2\x78\xd8\x02\xd1\xee\xaf\x13\x21" + "\xba\x59\x29\xdc", + "\x1f\x60\xb0\x1d", 1, 318, NULL, 0}, + {128, 32, 4, + "\xd3\x41\x9b\xe8\x21\x08\x7a\xcd\x02\x12\x3a\x92\x48\x03\x33\x59", + "\xc7\x59\x0e\xa9\xb8\x00\x00\x00\xbb\xb0\x57\x03\x88\x09\x49\x6b\xcf\xf8" + "\x6d\x6f\xbc\x8c\xe5\xb1\x35\xa0\x6b\x16\x60\x54\xf2\xd5\x65\xbe\x8a\xce" + "\x75\xdc\x85\x1e\x0b\xcd\xd8\xf0\x71\x41\xc4\x95\x87\x2f\xb5\xd8\xc0\xc6" + "\x6a\x8b\x6d\xa5\x56\x66\x3e\x4e\x46\x12\x05\xd8\x45\x80\xbe\xe5\xbc\x7e", + "\x68\x46\xa2\xf0", 1, 575, NULL, 0}, + {128, 32, 5, + "\x83\xfd\x23\xa2\x44\xa7\x4c\xf3\x58\xda\x30\x19\xf1\x72\x26\x35", + "\x36\xaf\x61\x44\x7c\x00\x00\x00\x35\xc6\x87\x16\x63\x3c\x66\xfb\x75\x0c" + "\x26\x68\x65\xd5\x3c\x11\xea\x05\xb1\xe9\xfa\x49\xc8\x39\x8d\x48\xe1\xef" + "\xa5\x90\x9d\x39\x47\x90\x28\x37\xf5\xae\x96\xd5\xa0\x5b\xc8\xd6\x1c\xa8" + "\xdb\xef\x1b\x13\xa4\xb4\xab\xfe\x4f\xb1\x00\x60\x45\xb6\x74\xbb\x54\x72" + "\x93\x04\xc3\x82\xbe\x53\xa5\xaf\x05\x55\x61\x76\xf6\xea\xa2\xef\x1d\x05" + "\xe4\xb0\x83\x18\x1e\xe6\x74\xcd\xa5\xa4\x85\xf7\x4d\x7a", + "\xe6\x57\xe1\x82", 1, 832, NULL, 0}, + {128, 32, 6, + "\x68\x32\xa6\x5c\xff\x44\x73\x62\x1e\xbd\xd4\xba\x26\xa9\x21\xfe", + "\x36\xaf\x61\x44\xc0\x00\x00\x00\xd3\xc5\x38\x39\x62\x68\x20\x71\x77\x65" + "\x66\x76\x20\x32\x38\x37\x63\x62\x40\x98\x1b\xa6\x82\x4c\x1b\xfb\x1a\xb4" + "\x85\x47\x20\x29\xb7\x1d\x80\x8c\xe3\x3e\x2c\xc3\xc0\xb5\xfc\x1f\x3d\xe8" + "\xa6\xdc", + "\xf0\x66\x8c\x1e", 1, 447, NULL, 0}, + {128, 32, 7, + "\x5d\x0a\x80\xd8\x13\x4a\xe1\x96\x77\x82\x4b\x67\x1e\x83\x8a\xf4", + "\x78\x27\xfa\xb2\x2c\x00\x00\x00\x70\xde\xdf\x2d\xc4\x2c\x5c\xbd\x3a\x96" + "\xf8\xa0\xb1\x14\x18\xb3\x60\x8d\x57\x33\x60\x4a\x2c\xd3\x6a\xab\xc7\x0c" + "\xe3\x19\x3b\xb5\x15\x3b\xe2\xd3\xc0\x6d\xfd\xb2\xd1\x6e\x9c\x35\x71\x58" + "\xbe\x6a\x41\xd6\xb8\x61\xe4\x91\xdb\x3f\xbf\xeb\x51\x8e\xfc\xf0\x48\xd7" + "\xd5\x89\x53\x73\x0f\xf3\x0c\x9e\xc4\x70\xff\xcd\x66\x3d\xc3\x42\x01\xc3" + "\x6a\xdd\xc0\x11\x1c\x35\xb3\x8a\xfe\xe7\xcf\xdb\x58\x2e\x37\x31\xf8\xb4" + "\xba\xa8\xd1\xa8\x9c\x06\xe8\x11\x99\xa9\x71\x62\x27\xbe\x34\x4e\xfc\xb4" + "\x36\xdd\xd0\xf0\x96\xc0\x64\xc3\xb5\xe2\xc3\x99\x99\x3f\xc7\x73\x94\xf9" + "\xe0\x97\x20\xa8\x11\x85\x0e\xf2\x3b\x2e\xe0\x5d\x9e\x61\x73\x60\x9d\x86" + "\xe1\xc0\xc1\x8e\xa5\x1a\x01\x2a\x00\xbb\x41\x3b\x9c\xb8\x18\x8a\x70\x3c" + "\xd6\xba\xe3\x1c\xc6\x7b\x34\xb1\xb0\x00\x19\xe6\xa2\xb2\xa6\x90\xf0\x26" + "\x71\xfe\x7c\x9e\xf8\xde\xc0\x09\x4e\x53\x37\x63\x47\x8d\x58\xd2\xc5\xf5" + "\xb8\x27\xa0\x14\x8c\x59\x48\xa9\x69\x31\xac\xf8\x4f\x46\x5a\x64\xe6\x2c" + "\xe7\x40\x07\xe9\x91\xe3\x7e\xa8\x23\xfa\x0f\xb2\x19\x23\xb7\x99\x05\xb7" + "\x33\xb6\x31\xe6\xc7\xd6\x86\x0a\x38\x31\xac\x35\x1a\x9c\x73\x0c\x52\xff" + "\x72\xd9\xd3\x08\xee\xdb\xab\x21\xfd\xe1\x43\xa0\xea\x17\xe2\x3e\xdc\x1f" + "\x74\xcb\xb3\x63\x8a\x20\x33\xaa\xa1\x54\x64\xea\xa7\x33\x38\x5d\xbb\xeb" + "\x6f\xd7\x35\x09\xb8\x57\xe6\xa4\x19\xdc\xa1\xd8\x90\x7a\xf9\x77\xfb\xac" + "\x4d\xfa\x35\xec", + "\xf4\xcc\x8f\xa3", 1, 2622, NULL, 0}, + {128, 32, 8, + "\xb3\x12\x0f\xfd\xb2\xcf\x6a\xf4\xe7\x3e\xaf\x2e\xf4\xeb\xec\x69", + "\x29\x6f\x39\x3c\x5c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01" + "\x01\x01\x01\x01\x01\x01\xe0\x95\x80\x45\xf3\xa0\xbb\xa4\xe3\x96\x83\x46" + "\xf0\xa3\xb8\xa7\xc0\x2a\x01\x8a\xe6\x40\x76\x52\x26\xb9\x87\xc9\x13\xe6" + "\xcb\xf0\x83\x57\x00\x16\xcf\x83\xef\xbc\x61\xc0\x82\x51\x3e\x21\x56\x1a" + "\x42\x7c\x00\x9d\x28\xc2\x98\xef\xac\xe7\x8e\xd6\xd5\x6c\x2d\x45\x05\xad" + "\x03\x2e\x9c\x04\xdc\x60\xe7\x3a\x81\x69\x6d\xa6\x65\xc6\xc4\x86\x03\xa5" + "\x7b\x45\xab\x33\x22\x15\x85\xe6\x8e\xe3\x16\x91\x87\xfb\x02\x39\x52\x86" + "\x32\xdd\x65\x6c\x80\x7e\xa3\x24\x8b\x7b\x46\xd0\x02\xb2\xb5\xc7\x45\x8e" + "\xb8\x5b\x9c\xe9\x58\x79\xe0\x34\x08\x59\x05\x5e\x3b\x0a\xbb\xc3\xea\xce" + "\x87\x19\xca\xa8\x02\x65\xc9\x72\x05\xd5\xdc\x4b\xcc\x90\x2f\xe1\x83\x96" + "\x29\xed\x71\x32\x8a\x0f\x04\x49\xf5\x88\x55\x7e\x68\x98\x86\x0e\x04\x2a" + "\xec\xd8\x4b\x24\x04\xc2\x12\xc9\x22\x2d\xa5\xbf\x8a\x89\xef\x67\x97\x87" + "\x0c\xf5\x07\x71\xa6\x0f\x66\xa2\xee\x62\x85\x36\x57\xad\xdf\x04\xcd\xde" + "\x07\xfa\x41\x4e\x11\xf1\x2b\x4d\x81\xb9\xb4\xe8\xac\x53\x8e\xa3\x06\x66" + "\x68\x8d\x88\x1f\x6c\x34\x84\x21\x99\x2f\x31\xb9\x4f\x88\x06\xed\x8f\xcc" + "\xff\x4c\x91\x23\xb8\x96\x42\x52\x7a\xd6\x13\xb1\x09\xbf\x75\x16\x74\x85" + "\xf1\x26\x8b\xf8\x84\xb4\xcd\x23\xd2\x9a\x09\x34\x92\x57\x03\xd6\x34\x09" + "\x8f\x77\x67\xf1\xbe\x74\x91\xe7\x08\xa8\xbb\x94\x9a\x38\x73\x70\x8a\xef" + "\x4a\x36\x23\x9e\x50\xcc\x08\x23\x5c\xd5\xed\x6b\xbe\x57\x86\x68\xa1\x7b" + "\x58\xc1\x17\x1d\x0b\x90\xe8\x13\xa9\xe4\xf5\x8a\x89\xd7\x19\xb1\x10\x42" + "\xd6\x36\x0b\x1b\x0f\x52\xde\xb7\x30\xa5\x8d\x58\xfa\xf4\x63\x15\x95\x4b" + "\x0a\x87\x26\x91\x47\x59\x77\xdc\x88\xc0\xd7\x33\xfe\xff\x54\x60\x0a\x0c" + "\xc1\xd0\x30\x0a\xaa\xeb\x94\x57\x2c\x6e\x95\xb0\x1a\xe9\x0d\xe0\x4f\x1d" + "\xce\x47\xf8\x7e\x8f\xa7\xbe\xbf\x77\xe1\xdb\xc2\x0d\x6b\xa8\x5c\xb9\x14" + "\x3d\x51\x8b\x28\x5d\xfa\x04\xb6\x98\xbf\x0c\xf7\x81\x9f\x20\xfa\x7a\x28" + "\x8e\xb0\x70\x3d\x99\x5c\x59\x94\x0c\x7c\x66\xde\x57\xa9\xb7\x0f\x82\x37" + "\x9b\x70\xe2\x03\x1e\x45\x0f\xcf\xd2\x18\x13\x26\xfc\xd2\x8d\x88\x23\xba" + "\xaa\x80\xdf\x6e\x0f\x44\x35\x59\x64\x75\x39\xfd\x89\x07\xc0\xff\xd9\xd7" + "\x9c\x13\x0e\xd8\x1c\x9a\xfd\x9b\x7e\x84\x8c\x9f\xed\x38\x44\x3d\x5d\x38" + "\x0e\x53\xfb\xdb\x8a\xc8\xc3\xd3\xf0\x68\x76\x05\x4f\x12\x24\x61\x10\x7d" + "\xe9\x2f\xea\x09\xc6\xf6\x92\x3a\x18\x8d\x53\xaf\xe5\x4a\x10\xf6\x0e\x6e" + "\x9d\x5a\x03\xd9\x96\xb5\xfb\xc8\x20\xf8\xa6\x37\x11\x6a\x27\xad\x04\xb4" + "\x44\xa0\x93\x2d\xd6\x0f\xbd\x12\x67\x1c\x11\xe1\xc0\xec\x73\xe7\x89\x87" + "\x9f\xaa\x3d\x42\xc6\x4d\x20\xcd\x12\x52\x74\x2a\x37\x68\xc2\x5a\x90\x15" + "\x85\x88\x8e\xce\xe1\xe6\x12\xd9\x93\x6b\x40\x3b\x07\x75\x94\x9a\x66\xcd" + "\xfd\x99\xa2\x9b\x13\x45\xba\xa8\xd9\xd5\x40\x0c\x91\x02\x4b\x0a\x60\x73" + "\x63\xb0\x13\xce\x5d\xe9\xae\x86\x9d\x3b\x8d\x95\xb0\x57\x0b\x3c\x2d\x39" + "\x14\x22\xd3\x24\x50\xcb\xcf\xae\x96\x65\x22\x86\xe9\x6d\xec\x12\x14\xa9" + "\x34\x65\x27\x98\x0a\x81\x92\xea\xc1\xc3\x9a\x3a\xaf\x6f\x15\x35\x1d\xa6" + "\xbe\x76\x4d\xf8\x97\x72\xec\x04\x07\xd0\x6e\x44\x15\xbe\xfa\xe7\xc9\x25" + "\x80\xdf\x9b\xf5\x07\x49\x7c\x8f\x29\x95\x16\x0d\x4e\x21\x8d\xaa\xcb\x02" + "\x94\x4a\xbf\x83\x34\x0c\xe8\xbe\x16\x86\xa9\x60\xfa\xf9\x0e\x2d\x90\xc5" + "\x5c\xc6\x47\x5b\xab\xc3\x17\x1a\x80\xa3\x63\x17\x49\x54\x95\x5d\x71\x01" + "\xda\xb1\x6a\xe8\x17\x91\x67\xe2\x14\x44\xb4\x43\xa9\xea\xaa\x7c\x91\xde" + "\x36\xd1\x18\xc3\x9d\x38\x9f\x8d\xd4\x46\x9a\x84\x6c\x9a\x26\x2b\xf7\xfa" + "\x18\x48\x7a\x79\xe8\xde\x11\x69\x9e\x0b\x8f\xdf\x55\x7c\xb4\x87\x19\xd4" + "\x53\xba\x71\x30\x56\x10\x9b\x93\xa2\x18\xc8\x96\x75\xac\x19\x5f\xb4\xfb" + "\x06\x63\x9b\x37\x97\x14\x49\x55\xb3\xc9\x32\x7d\x1a\xec\x00\x3d\x42\xec" + "\xd0\xea\x98\xab\xf1\x9f\xfb\x4a\xf3\x56\x1a\x67\xe7\x7c\x35\xbf\x15\xc5" + "\x9c\x24\x12\xda\x88\x1d\xb0\x2b\x1b\xfb\xce\xbf\xac\x51\x52\xbc\x99\xbc" + "\x3f\x1d\x15\xf7\x71\x00\x1b\x70\x29\xfe\xdb\x02\x8f\x8b\x85\x2b\xc4\x40" + "\x7e\xb8\x3f\x89\x1c\x9c\xa7\x33\x25\x4f\xdd\x1e\x9e\xdb\x56\x91\x9c\xe9" + "\xfe\xa2\x1c\x17\x40\x72\x52\x1c\x18\x31\x9a\x54\xb5\xd4\xef\xbe\xbd\xdf" + "\x1d\x8b\x69\xb1\xcb\xf2\x5f\x48\x9f\xcc\x98\x13\x72\x54\x7c\xf4\x1d\x00" + "\x8e\xf0\xbc\xa1\x92\x6f\x93\x4b\x73\x5e\x09\x0b\x3b\x25\x1e\xb3\x3a\x36" + "\xf8\x2e\xd9\xb2\x9c\xf4\xcb\x94\x41\x88\xfa\x0e\x1e\x38\xdd\x77\x8f\x7d" + "\x1c\x9d\x98\x7b\x28\xd1\x32\xdf\xb9\x73\x1f\xa4\xf4\xb4\x16\x93\x5b\xe4" + "\x9d\xe3\x05\x16\xaf\x35\x78\x58\x1f\x2f\x13\xf5\x61\xc0\x66\x33\x61\x94" + "\x1e\xab\x24\x9a\x4b\xc1\x23\xf8\xd1\x5c\xd7\x11\xa9\x56\xa1\xbf\x20\xfe" + "\x6e\xb7\x8a\xea\x23\x73\x36\x1d\xa0\x42\x6c\x79\xa5\x30\xc3\xbb\x1d\xe0" + "\xc9\x97\x22\xef\x1f\xde\x39\xac\x2b\x00\xa0\xa8\xee\x7c\x80\x0a\x08\xbc" + "\x22\x64\xf8\x9f\x4e\xff\xe6\x27\xac\x2f\x05\x31\xfb\x55\x4f\x6d\x21\xd7" + "\x4c\x59\x0a\x70\xad\xfa\xa3\x90\xbd\xfb\xb3\xd6\x8e\x46\x21\x5c\xab\x18" + "\x7d\x23\x68\xd5\xa7\x1f\x5e\xbe\xc0\x81\xcd\x3b\x20\xc0\x82\xdb\xe4\xcd" + "\x2f\xac\xa2\x87\x73\x79\x5d\x6b\x0c\x10\x20\x4b\x65\x9a\x93\x9e\xf2\x9b" + "\xbe\x10\x88\x24\x36\x24\x42\x99\x27\xa7\xeb\x57\x6d\xd3\xa0\x0e\xa5\xe0" + "\x1a\xf5\xd4\x75\x83\xb2\x27\x2c\x0c\x16\x1a\x80\x65\x21\xa1\x6f\xf9\xb0" + "\xa7\x22\xc0\xcf\x26\xb0\x25\xd5\x83\x6e\x22\x58\xa4\xf7\xd4\x77\x3a\xc8" + "\x01\xe4\x26\x3b\xc2\x94\xf4\x3d\xef\x7f\xa8\x70\x3f\x3a\x41\x97\x46\x35" + "\x25\x88\x76\x52\xb0\xb2\xa4\xa2\xa7\xcf\x87\xf0\x09\x14\x87\x1e\x25\x03" + "\x91\x13\xc7\xe1\x61\x8d\xa3\x40\x64\xb5\x7a\x43\xc4\x63\x24\x9f\xb8\xd0" + "\x5e\x0f\x26\xf4\xa6\xd8\x49\x72\xe7\xa9\x05\x48\x24\x14\x5f\x91\x29\x5c" + "\xdb\xe3\x9a\x6f\x92\x0f\xac\xc6\x59\x71\x2b\x46\xa5\x4b\xa2\x95\xbb\xe6" + "\xa9\x01\x54\xe9\x1b\x33\x98\x5a\x2b\xcd\x42\x0a\xd5\xc6\x7e\xc9\xad\x8e" + "\xb7\xac\x68\x64\xdb\x27\x2a\x51\x6b\xc9\x4c\x28\x39\xb0\xa8\x16\x9a\x6b" + "\xf5\x8e\x1a\x0c\x2a\xda\x8c\x88\x3b\x7b\xf4\x97\xa4\x91\x71\x26\x8e\xd1" + "\x5d\xdd\x29\x69\x38\x4e\x7f\xf4\xbf\x4a\xab\x2e\xc9\xec\xc6\x52\x9c\xf6" + "\x29\xe2\xdf\x0f\x08\xa7\x7a\x65\xaf\xa1\x2a\xa9\xb5\x05\xdf\x8b\x28\x7e" + "\xf6\xcc\x91\x49\x3d\x1c\xaa\x39\x07\x6e\x28\xef\x1e\xa0\x28\xf5\x11\x8d" + "\xe6\x1a\xe0\x2b\xb6\xae\xfc\x33\x43\xa0\x50\x29\x2f\x19\x9f\x40\x18\x57" + "\xb2\xbe\xad\x5e\x6e\xe2\xa1\xf1\x91\x02\x2f\x92\x78\x01\x6f\x04\x77\x91" + "\xa9\xd1\x8d\xa7\xd2\xa6\xd2\x7f\x2e\x0e\x51\xc2\xf6\xea\x30\xe8\xac\x49" + "\xa0\x60\x4f\x4c\x13\x54\x2e\x85\xb6\x83\x81\xb9\xfd\xcf\xa0\xce\x4b\x2d" + "\x34\x13\x54\x85\x2d\x36\x02\x45\xc5\x36\xb6\x12\xaf\x71\xf3\xe7\x7c\x90" + "\x95\xae\x2d\xbd\xe5\x04\xb2\x65\x73\x3d\xab\xfe\x10\xa2\x0f\xc7\xd6\xd3" + "\x2c\x21\xcc\xc7\x2b\x8b\x34\x44\xae\x66\x3d\x65\x92\x2d\x17\xf8\x2c\xaa" + "\x2b\x86\x5c\xd8\x89\x13\xd2\x91\xa6\x58\x99\x02\x6e\xa1\x32\x84\x39\x72" + "\x3c\x19\x8c\x36\xb0\xc3\xc8\xd0\x85\xbf\xaf\x8a\x32\x0f\xde\x33\x4b\x4a" + "\x49\x19\xb4\x4c\x2b\x95\xf6\xe8\xec\xf7\x33\x93\xf7\xf0\xd2\xa4\x0e\x60" + "\xb1\xd4\x06\x52\x6b\x02\x2d\xdc\x33\x18\x10\xb1\xa5\xf7\xc3\x47\xbd\x53" + "\xed\x1f\x10\x5d\x6a\x0d\x30\xab\xa4\x77\xe1\x78\x88\x9a\xb2\xec\x55\xd5" + "\x58\xde\xab\x26\x30\x20\x43\x36\x96\x2b\x4d\xb5\xb6\x63\xb6\x90\x2b\x89" + "\xe8\x5b\x31\xbc\x6a\xf5\x0f\xc5\x0a\xcc\xb3\xfb\x9b\x57\xb6\x63\x29\x70" + "\x31\x37\x8d\xb4\x78\x96\xd7\xfb\xaf\x6c\x60\x0a\xdd\x2c\x67\xf9\x36\xdb" + "\x03\x79\x86\xdb\x85\x6e\xb4\x9c\xf2\xdb\x3f\x7d\xa6\xd2\x36\x50\xe4\x38" + "\xf1\x88\x40\x41\xb0\x13\x11\x9e\x4c\x2a\xe5\xaf\x37\xcc\xcd\xfb\x68\x66" + "\x07\x38\xb5\x8b\x3c\x59\xd1\xc0\x24\x84\x37\x47\x2a\xba\x1f\x35\xca\x1f" + "\xb9\x0c\xd7\x14\xaa\x9f\x63\x55\x34\xf4\x9e\x7c\x5b\xba\x81\xc2\xb6\xb3" + "\x6f\xde\xe2\x1c\xa2\x7e\x34\x7f\x79\x3d\x2c\xe9\x44\xed\xb2\x3c\x8c\x9b" + "\x91\x4b\xe1\x03\x35\xe3\x50\xfe\xb5\x07\x03\x94\xb7\xa4\xa1\x5c\x0c\xa1" + "\x20\x28\x35\x68\xb7\xbf\xc2\x54\xfe\x83\x8b\x13\x7a\x21\x47\xce\x7c\x11" + "\x3a\x3a\x4d\x65\x49\x9d\x9e\x86\xb8\x7d\xbc\xc7\xf0\x3b\xbd\x3a\x3a\xb1" + "\xaa\x24\x3e\xce\x5b\xa9\xbc\xf2\x5f\x82\x83\x6c\xfe\x47\x3b\x2d\x83\xe7" + "\xa7\x20\x1c\xd0\xb9\x6a\x72\x45\x1e\x86\x3f\x6c\x3b\xa6\x64\xa6\xd0\x73" + "\xd1\xf7\xb5\xed\x99\x08\x65\xd9\x78\xbd\x38\x15\xd0\x60\x94\xfc\x9a\x2a" + "\xba\x52\x21\xc2\x2d\x5a\xb9\x96\x38\x9e\x37\x21\xe3\xaf\x5f\x05\xbe\xdd" + "\xc2\x87\x5e\x0d\xfa\xeb\x39\x02\x1e\xe2\x7a\x41\x18\x7c\xbb\x45\xef\x40" + "\xc3\xe7\x3b\xc0\x39\x89\xf9\xa3\x0d\x12\xc5\x4b\xa7\xd2\x14\x1d\xa8\xa8" + "\x75\x49\x3e\x65\x77\x6e\xf3\x5f\x97\xde\xbc\x22\x86\xcc\x4a\xf9\xb4\x62" + "\x3e\xee\x90\x2f\x84\x0c\x52\xf1\xb8\xad\x65\x89\x39\xae\xf7\x1f\x3f\x72" + "\xb9\xec\x1d\xe2\x15\x88\xbd\x35\x48\x4e\xa4\x44\x36\x34\x3f\xf9\x5e\xad" + "\x6a\xb1\xd8\xaf\xb1\xb2\xa3\x03\xdf\x1b\x71\xe5\x3c\x4a\xea\x6b\x2e\x3e" + "\x93\x72\xbe\x0d\x1b\xc9\x97\x98\xb0\xce\x3c\xc1\x0d\x2a\x59\x6d\x56\x5d" + "\xba\x82\xf8\x8c\xe4\xcf\xf3\xb3\x3d\x5d\x24\xe9\xc0\x83\x11\x24\xbf\x1a" + "\xd5\x4b\x79\x25\x32\x98\x3d\xd6\xc3\xa8\xb7\xd0", + "\xeb\xd5\xcc\xb0", 1, 16512, NULL, 0}, + {0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0} + }; diff --git a/test/crc_test.c b/test/kat-app/crc_test.c similarity index 99% rename from test/crc_test.c rename to test/kat-app/crc_test.c index a882d614a346cf0f9086be54983bb6017d0ff275..a9978913b268f982c19d4a8f49969bc83ea75e8e 100644 --- a/test/crc_test.c +++ b/test/kat-app/crc_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/ctr_test.c b/test/kat-app/ctr_test.c similarity index 97% rename from test/ctr_test.c rename to test/kat-app/ctr_test.c index 8ad8debec69359b1bc615b011f3256da4e8779ed..3252e7f8200f975853a847da88b1707032cae666 100644 --- a/test/ctr_test.c +++ b/test/kat-app/ctr_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -1509,6 +1509,7 @@ test_ctr_burst(struct IMB_MGR *mb_mgr, job->hash_alg = IMB_AUTH_NULL; job->user_data = targets[i]; job->user_data2 = (void *)((uint64_t)i); + imb_set_session(mb_mgr, job); } completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); @@ -1715,24 +1716,27 @@ test_ctr_vectors(struct IMB_MGR *mb_mgr, printf("AES-CTR standard test vectors:\n"); for (vect = 0; vect < vectors_cnt; vect++) { struct test_suite_context *ctx; + + if (!quiet_mode) { #ifdef DEBUG - if (alg == IMB_CIPHER_CNTR) - printf("Standard vector %u/%u Keylen:%d " - "IVlen:%d PTLen:%d\n", - vect, vectors_cnt - 1, - (int) vectors[vect].Klen, - (int) vectors[vect].IVlen, - (int) vectors[vect].Plen); - else - printf("Bit vector %u/%u Keylen:%d " - "IVlen:%d PTLen:%d\n", - vect, vectors_cnt - 1, - (int) vectors[vect].Klen, - (int) vectors[vect].IVlen, - (int) vectors[vect].Plen); + if (alg == IMB_CIPHER_CNTR) + printf("Standard vector %u/%u Keylen:%d " + "IVlen:%d PTLen:%d\n", + vect, vectors_cnt - 1, + (int) vectors[vect].Klen, + (int) vectors[vect].IVlen, + (int) vectors[vect].Plen); + else + printf("Bit vector %u/%u Keylen:%d " + "IVlen:%d PTLen:%d\n", + vect, vectors_cnt - 1, + (int) vectors[vect].Klen, + (int) vectors[vect].IVlen, + (int) vectors[vect].Plen); #else - printf("."); + printf("."); #endif + } switch (vectors[vect].Klen) { case IMB_KEY_128_BYTES: @@ -1824,7 +1828,8 @@ test_ctr_vectors(struct IMB_MGR *mb_mgr, } } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } static void @@ -1840,27 +1845,32 @@ test_ctr_vectors_burst(struct IMB_MGR *mb_mgr, DECLARE_ALIGNED(uint32_t expkey[4*15], 16); DECLARE_ALIGNED(uint32_t dust[4*15], 16); - printf("AES-CTR standard test vectors - Burst API:\n"); + if (!quiet_mode) + printf("AES-CTR standard test vectors - Burst API (N jobs = %u):\n", + num_jobs); for (vect = 0; vect < vectors_cnt; vect++) { struct test_suite_context *ctx; + + if (!quiet_mode) { #ifdef DEBUG - if (alg == IMB_CIPHER_CNTR) - printf("Standard vector %u/%u Keylen:%d " - "IVlen:%d PTLen:%d (burst)\n", - vect, vectors_cnt - 1, - (int) vectors[vect].Klen, - (int) vectors[vect].IVlen, - (int) vectors[vect].Plen); - else - printf("Bit vector %u/%u Keylen:%d " - "IVlen:%d PTLen:%d (burst)\n", - vect, vectors_cnt - 1, - (int) vectors[vect].Klen, - (int) vectors[vect].IVlen, - (int) vectors[vect].Plen); + if (alg == IMB_CIPHER_CNTR) + printf("Standard vector %u/%u Keylen:%d " + "IVlen:%d PTLen:%d (burst)\n", + vect, vectors_cnt - 1, + (int) vectors[vect].Klen, + (int) vectors[vect].IVlen, + (int) vectors[vect].Plen); + else + printf("Bit vector %u/%u Keylen:%d " + "IVlen:%d PTLen:%d (burst)\n", + vect, vectors_cnt - 1, + (int) vectors[vect].Klen, + (int) vectors[vect].IVlen, + (int) vectors[vect].Plen); #else - printf("."); + printf("."); #endif + } switch (vectors[vect].Klen) { case IMB_KEY_128_BYTES: @@ -2046,7 +2056,8 @@ test_ctr_vectors_burst(struct IMB_MGR *mb_mgr, } } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } int diff --git a/test/customop_test.c b/test/kat-app/customop_test.c similarity index 96% rename from test/customop_test.c rename to test/kat-app/customop_test.c index 382c2d1d1b0d5e596433bdb2a0cdfb2028eda8b3..70f1a9ed3ad66a8e7b56a652ed3d5381b6a46bed 100644 --- a/test/customop_test.c +++ b/test/kat-app/customop_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -36,11 +36,13 @@ #ifdef DEBUG #ifdef _WIN32 -#define TRACE(fmt, ...) fprintf(stderr, "%s:%d "fmt, \ - __FUNCTION__, __LINE__, __VA_ARGS__) +#define TRACE(fmt, ...) if (!quiet_mode) \ + fprintf(stderr, "%s:%d "fmt, \ + __FUNCTION__, __LINE__, __VA_ARGS__) #else -#define TRACE(fmt, ...) fprintf(stderr, "%s:%d "fmt, \ - __func__, __LINE__, __VA_ARGS__) +#define TRACE(fmt, ...) if (!quiet_mode) \ + fprintf(stderr, "%s:%d "fmt, \ + __func__, __LINE__, __VA_ARGS__) #endif #else # define TRACE(fmt, ...) diff --git a/test/customop_test.h b/test/kat-app/customop_test.h similarity index 97% rename from test/customop_test.h rename to test/kat-app/customop_test.h index 464641faf680d4b2a220cbcce13ab30d0337c4ea..fb5ca099cf7837368e25d360d75a1403ff5b4995 100644 --- a/test/customop_test.h +++ b/test/kat-app/customop_test.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/des_test.c b/test/kat-app/des_test.c similarity index 85% rename from test/des_test.c rename to test/kat-app/des_test.c index eb24b2e971b37de6b9f746721444d23d10065379..c4f08bc63479a1bc61523e1f7896d0f4bd0a8466 100644 --- a/test/des_test.c +++ b/test/kat-app/des_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -158,6 +158,10 @@ static struct des_vector docsis_vectors[] = { {DK3, DIV3, DP3, sizeof(DP3), DC3}, }; +static struct des_vector des_cfb_vectors[] = { + {DK3, DIV3, DP3, sizeof(DP3), DC3}, +}; + /* 3DES vectors - 2x and 3x keys */ static const uint8_t D3K1_1[] = { @@ -473,13 +477,16 @@ test_des_vectors(struct IMB_MGR *mb_mgr, printf("%s:\n", banner); for (vect = 0; vect < vec_cnt; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("Standard vector %d/%d PTLen:%d\n", - vect + 1, vec_cnt, - (int) vec_tab[vect].Plen); + printf("Standard vector %d/%d PTLen:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].Plen); #else - printf("."); + printf("."); #endif + } + des_key_schedule(ks, vec_tab[vect].K); if (test_des(mb_mgr, ks, NULL, NULL, @@ -530,7 +537,8 @@ test_des_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } static void @@ -547,13 +555,15 @@ test_des3_vectors(struct IMB_MGR *mb_mgr, printf("%s:\n", banner); for (vect = 0; vect < vec_cnt; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("Standard vector %d/%d PTLen:%d\n", - vect + 1, vec_cnt, - (int) vec_tab[vect].Plen); + printf("Standard vector %d/%d PTLen:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].Plen); #else - printf("."); + printf("."); #endif + } des_key_schedule(ks1, vec_tab[vect].K1); des_key_schedule(ks2, vec_tab[vect].K2); des_key_schedule(ks3, vec_tab[vect].K3); @@ -606,7 +616,79 @@ test_des3_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); +} + +static int +des_cfb_validate(struct test_suite_context *ctx) +{ + unsigned i; + + printf("DES-CFB standard test vectors:\n"); + for (i = 0; i < DIM(des_cfb_vectors); i++) { + uint8_t output1[8]; + uint8_t output2[8]; + uint64_t ks[16]; + + des_key_schedule(ks, des_cfb_vectors[i].K); + + /* Out of place */ + + /* encrypt test */ + des_cfb_one(output1, des_cfb_vectors[i].P, + (const uint64_t *)des_cfb_vectors[i].IV, ks, + (int)des_cfb_vectors[i].Plen); + if (memcmp(output1, des_cfb_vectors[i].C, + des_cfb_vectors[i].Plen)) { + printf("DES-CFB enc (OOP) vector %d mismatched\n", i); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + /* decrypt test */ + des_cfb_one(output2, output1, + (const uint64_t *)des_cfb_vectors[i].IV, ks, + (int)des_cfb_vectors[i].Plen); + if (memcmp(output2, des_cfb_vectors[i].P, + des_cfb_vectors[i].Plen)) { + printf("DES-CFB dec (OOP) vector %d mismatched\n", i); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + /* In place */ + + /* encrypt test */ + memcpy(output1, des_cfb_vectors[i].P, des_cfb_vectors[i].Plen); + des_cfb_one(output1, output1, + (const uint64_t *)des_cfb_vectors[i].IV, ks, + (int)des_cfb_vectors[i].Plen); + if (memcmp(output1, des_cfb_vectors[i].C, + des_cfb_vectors[i].Plen)) { + printf("DES-CFB enc (OOP) vector %d mismatched\n", i); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + /* decrypt test */ + memcpy(output1, des_cfb_vectors[i].C, des_cfb_vectors[i].Plen); + des_cfb_one(output1, output1, + (const uint64_t *)des_cfb_vectors[i].IV, ks, + (int)des_cfb_vectors[i].Plen); + if (memcmp(output1, des_cfb_vectors[i].P, + des_cfb_vectors[i].Plen)) { + printf("DES-CFB dec (OOP) vector %d mismatched\n", i); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + } + if (!quiet_mode) + printf("\n"); + return 1; } int @@ -626,6 +708,10 @@ des_test(struct IMB_MGR *mb_mgr) IMB_CIPHER_DOCSIS_DES, &ctx); errors += test_suite_end(&ctx); + test_suite_start(&ctx, "DES-CFB-64"); + des_cfb_validate(&ctx); + errors += test_suite_end(&ctx); + test_suite_start(&ctx, "3DES-CBC-192"); test_des_vectors(mb_mgr, DIM(vectors), vectors, "3DES (single key) standard test vectors", diff --git a/test/direct_api_param_test.c b/test/kat-app/direct_api_param_test.c similarity index 99% rename from test/direct_api_param_test.c rename to test/kat-app/direct_api_param_test.c index 0ff0c20c6cfc3b1842f47bf796e0d405c9a65eed..93ee734a2660605eaa601952a5aed9343faa7ecc 100644 --- a/test/direct_api_param_test.c +++ b/test/kat-app/direct_api_param_test.c @@ -3,7 +3,7 @@ *****************************************************************************/ /***************************************************************************** - Copyright (c) 2021-2022, Intel Corporation + Copyright (c) 2021-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -85,6 +85,7 @@ seg_handler(int signum) } #endif /* DEBUG */ +#ifndef __aarch64__ /* * @brief Performs direct API invalid param tests for IMB_AES_KEYEXP_128 */ static int @@ -1965,6 +1966,7 @@ test_IMB_AES256_GCM_PRE(struct IMB_MGR *mgr) } return 0; } +#endif /* __aarch64__ */ /* * @brief Performs direct API invalid param tests for IMB_ZUC_EEA3_1_BUFFER */ @@ -2198,6 +2200,7 @@ test_IMB_ZUC_EIA3_1_BUFFER(struct IMB_MGR *mgr) return 0; } +#ifndef __aarch64__ /* * @brief Performs direct API invalid param tests for IMB_KASUMI_F8_1_BUFFER */ static int @@ -2734,6 +2737,7 @@ test_IMB_KASUMI_INIT_F9_KEY_SCHED(struct IMB_MGR *mgr) } return 0; } +#endif /* __aarch64__ */ /* * @brief Performs direct API invalid param tests for IMB_SNOW3G_F8_1_BUFFER_BIT @@ -3931,6 +3935,7 @@ test_IMB_SNOW3G_INIT_KEY_SCHED(struct IMB_MGR *mgr) return 0; } +#ifndef __aarch64__ /* * @brief Performs direct API invalid param tests for IMB_GHASH */ static int @@ -3975,6 +3980,7 @@ test_IMB_GHASH(struct IMB_MGR *mgr) } return 0; } +#endif /* __aarch64__ */ /* * @brief Performs direct API invalid param tests for IMB_ZUC_EIA3_N_BUFFER */ @@ -4050,6 +4056,7 @@ test_IMB_ZUC_EIA3_N_BUFFER(struct IMB_MGR *mgr) return 0; } +#ifndef __aarch64__ /* * @brief Performs direct API invalid param tests for IMB_AES128_GCM_INIT_VAR_IV */ @@ -5234,6 +5241,7 @@ test_IMB_CHACHA20_POLY1305_DEC_FINALIZE(struct IMB_MGR *mgr) } return 0; } +#endif /* __aarch64__ */ int direct_api_param_test(struct IMB_MGR *mb_mgr) @@ -5259,6 +5267,7 @@ direct_api_param_test(struct IMB_MGR *mb_mgr) "skipping remaining tests\n"); goto dir_api_exit; } +#ifndef __aarch64__ errors += test_IMB_AES_KEYEXP_128(mb_mgr); run++; @@ -5384,6 +5393,7 @@ direct_api_param_test(struct IMB_MGR *mb_mgr) errors += test_IMB_AES256_GCM_PRE(mb_mgr); run++; +#endif /* __aarch64__ */ errors += test_IMB_ZUC_EEA3_1_BUFFER(mb_mgr); run++; @@ -5397,6 +5407,7 @@ direct_api_param_test(struct IMB_MGR *mb_mgr) errors += test_IMB_ZUC_EIA3_1_BUFFER(mb_mgr); run++; +#ifndef __aarch64__ errors += test_IMB_KASUMI_F8_1_BUFFER(mb_mgr); run++; @@ -5426,6 +5437,7 @@ direct_api_param_test(struct IMB_MGR *mb_mgr) errors += test_IMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr); run++; +#endif /* __aarch64__ */ errors += test_IMB_SNOW3G_F8_1_BUFFER_BIT(mb_mgr); run++; @@ -5457,12 +5469,15 @@ direct_api_param_test(struct IMB_MGR *mb_mgr) errors += test_IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr); run++; +#ifndef __aarch64__ errors += test_IMB_GHASH(mb_mgr); run++; +#endif /* __aarch64__ */ errors += test_IMB_ZUC_EIA3_N_BUFFER(mb_mgr); run++; +#ifndef __aarch64__ errors += test_IMB_AES128_GCM_INIT_VAR_IV(mb_mgr); run++; @@ -5552,6 +5567,7 @@ direct_api_param_test(struct IMB_MGR *mb_mgr) errors += test_IMB_CHACHA20_POLY1305_DEC_FINALIZE(mb_mgr); run++; +#endif /* __aarch64__ */ test_suite_update(&ts, run - errors, errors); dir_api_exit: diff --git a/test/direct_api_param_test.c_template b/test/kat-app/direct_api_param_test.c_template similarity index 98% rename from test/direct_api_param_test.c_template rename to test/kat-app/direct_api_param_test.c_template index b05afdee684bf93cab7a673bd4cc7aa300cfd302..fba1dcdc680b50d247716030e2598c78b2b4df70 100644 --- a/test/direct_api_param_test.c_template +++ b/test/kat-app/direct_api_param_test.c_template @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2021-2022, Intel Corporation + Copyright (c) 2021-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/direct_api_test.c b/test/kat-app/direct_api_test.c similarity index 93% rename from test/direct_api_test.c rename to test/kat-app/direct_api_test.c index 22d87b4e8a572eed160d26397da39f9232ec4ca0..fa746183c7e00c5a7d476f7f72ffce0d079e7864 100644 --- a/test/direct_api_test.c +++ b/test/kat-app/direct_api_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -33,10 +33,7 @@ #include #include - -#ifdef __x86_64__ #include "gcm_ctr_vectors_test.h" -#endif /* __x86_64__ */ #include "utils.h" @@ -68,6 +65,12 @@ seg_handler(int signum) } #endif /* DEBUG */ +static void print_progress(void) +{ + if (!quiet_mode) + printf("."); +} + #ifndef __aarch64__ /* * @brief Performs direct GCM API invalid param tests @@ -107,7 +110,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_ENC(mgr, NULL, NULL, NULL, NULL, -1, NULL, NULL, -1, NULL, -1); @@ -118,7 +121,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_ENC(mgr, NULL, NULL, NULL, NULL, -1, NULL, NULL, -1, NULL, -1); @@ -129,7 +132,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); /* GCM Decrypt API tests */ IMB_AES128_GCM_DEC(mgr, NULL, NULL, NULL, NULL, -1, @@ -141,7 +144,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_ENC(mgr, NULL, NULL, NULL, NULL, -1, NULL, NULL, -1, NULL, -1); @@ -152,7 +155,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_DEC(mgr, NULL, NULL, NULL, NULL, -1, NULL, NULL, -1, NULL, -1); @@ -163,7 +166,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); /* GCM Init tests */ IMB_AES128_GCM_INIT(mgr, NULL, NULL, NULL, NULL, -1); @@ -174,7 +177,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_INIT(mgr, NULL, NULL, NULL, NULL, -1); IMB_AES192_GCM_INIT(mgr, NULL, (struct gcm_context_data *)out_buf, @@ -184,7 +187,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_INIT(mgr, NULL, NULL, NULL, NULL, -1); IMB_AES256_GCM_INIT(mgr, NULL, (struct gcm_context_data *)out_buf, @@ -194,7 +197,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); /* GCM Encrypt update tests */ IMB_AES128_GCM_ENC_UPDATE(mgr, NULL, NULL, NULL, NULL, -1); @@ -204,7 +207,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_ENC_UPDATE(mgr, NULL, NULL, NULL, NULL, -1); IMB_AES192_GCM_ENC_UPDATE(mgr, NULL, NULL, out_buf, zero_buf, -1); @@ -213,7 +216,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_ENC_UPDATE(mgr, NULL, NULL, NULL, NULL, -1); IMB_AES256_GCM_ENC_UPDATE(mgr, NULL, NULL, out_buf, zero_buf, -1); @@ -222,7 +225,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); /* GCM Decrypt update tests */ IMB_AES128_GCM_DEC_UPDATE(mgr, NULL, NULL, NULL, NULL, -1); @@ -232,7 +235,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_DEC_UPDATE(mgr, NULL, NULL, NULL, NULL, -1); IMB_AES192_GCM_DEC_UPDATE(mgr, NULL, NULL, out_buf, zero_buf, -1); @@ -241,7 +244,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_DEC_UPDATE(mgr, NULL, NULL, NULL, NULL, -1); IMB_AES256_GCM_DEC_UPDATE(mgr, NULL, NULL, out_buf, zero_buf, -1); @@ -250,7 +253,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); /* GCM Encrypt complete tests */ IMB_AES128_GCM_ENC_FINALIZE(mgr, NULL, NULL, NULL, -1); @@ -260,7 +263,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_ENC_FINALIZE(mgr, NULL, NULL, NULL, -1); IMB_AES192_GCM_ENC_FINALIZE(mgr, NULL, NULL, out_buf, -1); @@ -269,7 +272,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_ENC_FINALIZE(mgr, NULL, NULL, NULL, -1); IMB_AES256_GCM_ENC_FINALIZE(mgr, NULL, NULL, out_buf, -1); @@ -278,7 +281,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); /* GCM Decrypt complete tests */ IMB_AES128_GCM_DEC_FINALIZE(mgr, NULL, NULL, NULL, -1); @@ -288,7 +291,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_DEC_FINALIZE(mgr, NULL, NULL, NULL, -1); IMB_AES192_GCM_DEC_FINALIZE(mgr, NULL, NULL, out_buf, -1); @@ -297,7 +300,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_DEC_FINALIZE(mgr, NULL, NULL, NULL, -1); IMB_AES256_GCM_DEC_FINALIZE(mgr, NULL, NULL, out_buf, -1); @@ -306,17 +309,17 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); /* GCM key data pre-processing tests */ IMB_AES128_GCM_PRECOMP(mgr, NULL); - printf("."); + print_progress(); IMB_AES192_GCM_PRECOMP(mgr, NULL); - printf("."); + print_progress(); IMB_AES256_GCM_PRECOMP(mgr, NULL); - printf("."); + print_progress(); IMB_AES128_GCM_PRE(mgr, NULL, NULL); IMB_AES128_GCM_PRE(mgr, NULL, key_data); @@ -325,7 +328,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES192_GCM_PRE(mgr, NULL, NULL); IMB_AES192_GCM_PRE(mgr, NULL, key_data); @@ -334,7 +337,7 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES256_GCM_PRE(mgr, NULL, NULL); IMB_AES256_GCM_PRE(mgr, NULL, key_data); @@ -343,9 +346,10 @@ test_gcm_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -384,7 +388,7 @@ test_key_exp_gen_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES_KEYEXP_192(mgr, NULL, NULL, NULL); IMB_AES_KEYEXP_192(mgr, NULL, out_buf, zero_buf); @@ -393,7 +397,7 @@ test_key_exp_gen_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES_KEYEXP_256(mgr, NULL, NULL, NULL); IMB_AES_KEYEXP_256(mgr, NULL, out_buf, zero_buf); @@ -402,7 +406,7 @@ test_key_exp_gen_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES_CMAC_SUBKEY_GEN_128(mgr, NULL, NULL, NULL); IMB_AES_CMAC_SUBKEY_GEN_128(mgr, NULL, out_buf, zero_buf); @@ -411,7 +415,7 @@ test_key_exp_gen_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_AES_XCBC_KEYEXP(mgr, NULL, NULL, NULL, NULL); IMB_AES_XCBC_KEYEXP(mgr, NULL, out_buf, out_buf, out_buf); @@ -420,7 +424,7 @@ test_key_exp_gen_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_DES_KEYSCHED(mgr, NULL, NULL); IMB_DES_KEYSCHED(mgr, (uint64_t *)out_buf, NULL); @@ -429,9 +433,10 @@ test_key_exp_gen_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -469,7 +474,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA1(mgr, NULL, -1, NULL); IMB_SHA1(mgr, NULL, BUF_SIZE, out_buf); @@ -478,7 +483,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA224_ONE_BLOCK(mgr, NULL, NULL); IMB_SHA224_ONE_BLOCK(mgr, NULL, out_buf); @@ -487,7 +492,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA224(mgr, NULL, -1, NULL); IMB_SHA224(mgr, NULL, BUF_SIZE, out_buf); @@ -496,7 +501,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA256_ONE_BLOCK(mgr, NULL, NULL); IMB_SHA256_ONE_BLOCK(mgr, NULL, out_buf); @@ -505,7 +510,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA256(mgr, NULL, -1, NULL); IMB_SHA256(mgr, NULL, BUF_SIZE, out_buf); @@ -514,7 +519,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA384_ONE_BLOCK(mgr, NULL, NULL); IMB_SHA384_ONE_BLOCK(mgr, NULL, out_buf); @@ -523,7 +528,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA384(mgr, NULL, -1, NULL); IMB_SHA384(mgr, NULL, BUF_SIZE, out_buf); @@ -532,7 +537,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA512_ONE_BLOCK(mgr, NULL, NULL); IMB_SHA512_ONE_BLOCK(mgr, NULL, out_buf); @@ -541,7 +546,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SHA512(mgr, NULL, -1, NULL); IMB_SHA512(mgr, NULL, BUF_SIZE, out_buf); @@ -550,7 +555,7 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_MD5_ONE_BLOCK(mgr, NULL, NULL); IMB_MD5_ONE_BLOCK(mgr, NULL, out_buf); @@ -559,9 +564,10 @@ test_hash_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -599,12 +605,117 @@ test_aes_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } +#endif /* __aarch64__ */ +/* + * @brief Performs direct ZUC API invalid param tests + */ +static int +test_zuc_api(struct IMB_MGR *mgr) +{ + const uint32_t text_len = BUF_SIZE; + const uint32_t inv_len = -1; + uint8_t out_buf[BUF_SIZE]; + uint8_t zero_buf[BUF_SIZE]; + int i, ret1, ret2, seg_err; /* segfault flag */ + void *out_bufs[NUM_BUFS]; + uint32_t lens[NUM_BUFS]; + + seg_err = setjmp(env); + if (seg_err) { + printf("%s: segfault occurred!\n", __func__); + return 1; + } + + for (i = 0; i < NUM_BUFS; i++) { + out_bufs[i] = (void *)&out_buf; + lens[i] = text_len; + } + + memset(out_buf, 0, text_len); + memset(zero_buf, 0, text_len); + + /** + * API are generally tested twice: + * 1. test with all invalid params + * 2. test with some valid params (in, out, len) + * and verify output buffer is not modified + */ + + ret1 = zuc_eea3_iv_gen(inv_len, (const uint8_t)inv_len, + (const uint8_t)inv_len, NULL); + ret2 = zuc_eea3_iv_gen(inv_len, (const uint8_t)inv_len, + (const uint8_t)inv_len, out_buf); + if ((memcmp(out_buf, zero_buf, text_len) != 0) || + ret1 == 0 || ret2 == 0) { + printf("%s: zuc_eea3_iv_gen, invalid " + "param test failed!\n", __func__); + return 1; + } + print_progress(); + + ret1 = zuc_eia3_iv_gen(inv_len, (const uint8_t)inv_len, + (const uint8_t)inv_len, NULL); + ret2 = zuc_eia3_iv_gen(inv_len, (const uint8_t)inv_len, + (const uint8_t)inv_len, out_buf); + if ((memcmp(out_buf, zero_buf, text_len) != 0) || + ret1 == 0 || ret2 == 0) { + printf("%s: zuc_eia3_iv_gen, invalid " + "param test failed!\n", __func__); + return 1; + } + print_progress(); + + IMB_ZUC_EEA3_1_BUFFER(mgr, NULL, NULL, NULL, NULL, inv_len); + IMB_ZUC_EEA3_1_BUFFER(mgr, NULL, NULL, NULL, out_buf, text_len); + if (memcmp(out_buf, zero_buf, text_len) != 0) { + printf("%s: IMB_ZUC_EEA3_1_BUFFER, invalid " + "param test failed!\n", __func__); + return 1; + } + print_progress(); + + IMB_ZUC_EEA3_4_BUFFER(mgr, NULL, NULL, NULL, NULL, NULL); + IMB_ZUC_EEA3_4_BUFFER(mgr, NULL, NULL, NULL, out_bufs, lens); + if (memcmp(out_buf, zero_buf, text_len) != 0) { + printf("%s: IMB_ZUC_EEA3_4_BUFFER, invalid " + "param test failed!\n", __func__); + return 1; + } + print_progress(); + + IMB_ZUC_EEA3_N_BUFFER(mgr, NULL, NULL, NULL, + NULL, NULL, inv_len); + IMB_ZUC_EEA3_N_BUFFER(mgr, NULL, NULL, NULL, + out_bufs, lens, NUM_BUFS); + if (memcmp(out_buf, zero_buf, text_len) != 0) { + printf("%s: IMB_ZUC_EEA3_N_BUFFER, invalid " + "param test failed!\n", __func__); + return 1; + } + print_progress(); + + IMB_ZUC_EIA3_1_BUFFER(mgr, NULL, NULL, NULL, inv_len, NULL); + IMB_ZUC_EIA3_1_BUFFER(mgr, NULL, NULL, NULL, text_len, out_bufs[0]); + if (memcmp(out_buf, zero_buf, text_len) != 0) { + printf("%s: IMB_ZUC_EIA3_1_BUFFER, invalid " + "param test failed!\n", __func__); + return 1; + } + print_progress(); + + if (!quiet_mode) + printf("\n"); + return 0; +} + +#ifndef __aarch64__ /* * @brief Performs direct KASUMI API invalid param tests */ @@ -651,7 +762,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); ret1 = kasumi_f9_iv_gen(inv_len, inv_len, NULL); if ((memcmp(out_buf, zero_buf, text_len) != 0) || ret1 == 0) { @@ -659,7 +770,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F8_1_BUFFER(mgr, NULL, inv_iv, NULL, NULL, inv_len); IMB_KASUMI_F8_1_BUFFER(mgr, NULL, inv_iv, NULL, out_buf, text_len); @@ -668,7 +779,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F8_1_BUFFER_BIT(mgr, NULL, inv_iv, NULL, NULL, inv_len, inv_len); @@ -679,7 +790,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F8_2_BUFFER(mgr, NULL, inv_iv, inv_iv, NULL, NULL, inv_len, NULL, NULL, inv_len); @@ -690,7 +801,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F8_3_BUFFER(mgr, NULL, inv_iv, inv_iv, inv_iv, NULL, NULL, NULL, NULL, NULL, NULL, inv_len); @@ -701,7 +812,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F8_4_BUFFER(mgr, NULL, inv_iv, inv_iv, inv_iv, inv_iv, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -714,7 +825,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F8_N_BUFFER(mgr, NULL, NULL, NULL, NULL, NULL, inv_len); @@ -725,7 +836,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F9_1_BUFFER(mgr, NULL, NULL, inv_len, NULL); IMB_KASUMI_F9_1_BUFFER(mgr, NULL, NULL, text_len, out_buf); @@ -734,7 +845,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_KASUMI_F9_1_BUFFER_USER(mgr, NULL, inv_iv, NULL, inv_len, NULL, inv_len); @@ -745,7 +856,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); ret1 = IMB_KASUMI_INIT_F8_KEY_SCHED(mgr, NULL, NULL); ret2 = IMB_KASUMI_INIT_F8_KEY_SCHED(mgr, NULL, @@ -756,7 +867,7 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); ret1 = IMB_KASUMI_INIT_F9_KEY_SCHED(mgr, NULL, NULL); ret2 = IMB_KASUMI_INIT_F9_KEY_SCHED(mgr, NULL, @@ -767,121 +878,21 @@ test_kasumi_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - if (IMB_KASUMI_KEY_SCHED_SIZE(mgr) <= 0) { + if (IMB_KASUMI_KEY_SCHED_SIZE(mgr) == 0) { printf("%s: IMB_KASUMI_KEY_SCHED_SIZE, invalid " "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } #endif /* __aarch64__ */ -/* - * @brief Performs direct ZUC API invalid param tests - */ -static int -test_zuc_api(struct IMB_MGR *mgr) -{ - const uint32_t text_len = BUF_SIZE; - const uint32_t inv_len = -1; - uint8_t out_buf[BUF_SIZE]; - uint8_t zero_buf[BUF_SIZE]; - int i, ret1, ret2, seg_err; /* segfault flag */ - void *out_bufs[NUM_BUFS]; - uint32_t lens[NUM_BUFS]; - - seg_err = setjmp(env); - if (seg_err) { - printf("%s: segfault occurred!\n", __func__); - return 1; - } - - for (i = 0; i < NUM_BUFS; i++) { - out_bufs[i] = (void *)&out_buf; - lens[i] = text_len; - } - - memset(out_buf, 0, text_len); - memset(zero_buf, 0, text_len); - - /** - * API are generally tested twice: - * 1. test with all invalid params - * 2. test with some valid params (in, out, len) - * and verify output buffer is not modified - */ - - ret1 = zuc_eea3_iv_gen(inv_len, (const uint8_t)inv_len, - (const uint8_t)inv_len, NULL); - ret2 = zuc_eea3_iv_gen(inv_len, (const uint8_t)inv_len, - (const uint8_t)inv_len, out_buf); - if ((memcmp(out_buf, zero_buf, text_len) != 0) || - ret1 == 0 || ret2 == 0) { - printf("%s: zuc_eea3_iv_gen, invalid " - "param test failed!\n", __func__); - return 1; - } - printf("."); - - ret1 = zuc_eia3_iv_gen(inv_len, (const uint8_t)inv_len, - (const uint8_t)inv_len, NULL); - ret2 = zuc_eia3_iv_gen(inv_len, (const uint8_t)inv_len, - (const uint8_t)inv_len, out_buf); - if ((memcmp(out_buf, zero_buf, text_len) != 0) || - ret1 == 0 || ret2 == 0) { - printf("%s: zuc_eia3_iv_gen, invalid " - "param test failed!\n", __func__); - return 1; - } - printf("."); - - IMB_ZUC_EEA3_1_BUFFER(mgr, NULL, NULL, NULL, NULL, inv_len); - IMB_ZUC_EEA3_1_BUFFER(mgr, NULL, NULL, NULL, out_buf, text_len); - if (memcmp(out_buf, zero_buf, text_len) != 0) { - printf("%s: IMB_ZUC_EEA3_1_BUFFER, invalid " - "param test failed!\n", __func__); - return 1; - } - printf("."); - - IMB_ZUC_EEA3_4_BUFFER(mgr, NULL, NULL, NULL, NULL, NULL); - IMB_ZUC_EEA3_4_BUFFER(mgr, NULL, NULL, NULL, out_bufs, lens); - if (memcmp(out_buf, zero_buf, text_len) != 0) { - printf("%s: IMB_ZUC_EEA3_4_BUFFER, invalid " - "param test failed!\n", __func__); - return 1; - } - printf("."); - - IMB_ZUC_EEA3_N_BUFFER(mgr, NULL, NULL, NULL, - NULL, NULL, inv_len); - IMB_ZUC_EEA3_N_BUFFER(mgr, NULL, NULL, NULL, - out_bufs, lens, NUM_BUFS); - if (memcmp(out_buf, zero_buf, text_len) != 0) { - printf("%s: IMB_ZUC_EEA3_N_BUFFER, invalid " - "param test failed!\n", __func__); - return 1; - } - printf("."); - - IMB_ZUC_EIA3_1_BUFFER(mgr, NULL, NULL, NULL, inv_len, NULL); - IMB_ZUC_EIA3_1_BUFFER(mgr, NULL, NULL, NULL, text_len, out_bufs[0]); - if (memcmp(out_buf, zero_buf, text_len) != 0) { - printf("%s: IMB_ZUC_EIA3_1_BUFFER, invalid " - "param test failed!\n", __func__); - return 1; - } - printf("."); - - printf("\n"); - return 0; -} - /* * @brief Performs direct SNOW3G API invalid param tests */ @@ -927,7 +938,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); ret1 = snow3g_f9_iv_gen(inv_len, (const uint8_t)inv_len, (const uint8_t)inv_len, NULL); @@ -939,7 +950,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_1_BUFFER(mgr, NULL, NULL, NULL, NULL, inv_len); IMB_SNOW3G_F8_1_BUFFER(mgr, NULL, NULL, NULL, out_buf, text_len); @@ -948,7 +959,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_1_BUFFER_BIT(mgr, NULL, NULL, NULL, NULL, inv_len, inv_len); @@ -959,7 +970,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_2_BUFFER(mgr, NULL, NULL, NULL, NULL, NULL, inv_len, NULL, NULL, inv_len); @@ -970,7 +981,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_4_BUFFER(mgr, NULL, NULL, NULL, NULL, NULL, NULL, NULL, inv_len, NULL, NULL, inv_len, @@ -983,7 +994,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_8_BUFFER(mgr, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -1002,7 +1013,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(mgr, NULL, NULL, NULL, NULL, &inv_len); IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(mgr, NULL, NULL, NULL, out_bufs, lens); @@ -1011,7 +1022,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_N_BUFFER(mgr, NULL, NULL, NULL, NULL, NULL, inv_len); IMB_SNOW3G_F8_N_BUFFER(mgr, NULL, NULL, NULL, out_bufs, lens, NUM_BUFS); @@ -1020,7 +1031,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F8_N_BUFFER_MULTIKEY(mgr, NULL, NULL, NULL, NULL, NULL, inv_len); @@ -1031,7 +1042,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); IMB_SNOW3G_F9_1_BUFFER(mgr, NULL, NULL, NULL, inv_len, NULL); IMB_SNOW3G_F9_1_BUFFER(mgr, NULL, NULL, NULL, text_len, out_buf); @@ -1040,7 +1051,7 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); ret1 = IMB_SNOW3G_INIT_KEY_SCHED(mgr, NULL, NULL); ret2 = IMB_SNOW3G_INIT_KEY_SCHED(mgr, NULL, @@ -1051,16 +1062,17 @@ test_snow3g_api(struct IMB_MGR *mgr) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - if (IMB_SNOW3G_KEY_SCHED_SIZE(mgr) <= 0) { + if (IMB_SNOW3G_KEY_SCHED_SIZE(mgr) == 0) { printf("%s: IMB_SNOW3G_KEY_SCHED_SIZE, invalid " "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -1097,7 +1109,7 @@ test_clear_mem_api(void) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); imb_clear_mem(out_buf, 0); if (memcmp(out_buf, cmp_buf, text_len) != 0) { @@ -1105,7 +1117,7 @@ test_clear_mem_api(void) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); imb_clear_mem(out_buf, text_len); if (memcmp(out_buf, cmp_buf, text_len) == 0) { @@ -1113,9 +1125,10 @@ test_clear_mem_api(void) "param test failed!\n", __func__); return 1; } - printf("."); + print_progress(); - printf("\n"); + if (!quiet_mode) + printf("\n"); return 0; } @@ -1147,7 +1160,7 @@ direct_api_test(struct IMB_MGR *mb_mgr) goto dir_api_exit; } -#ifdef __x86_64__ +#ifndef __aarch64__ errors += test_gcm_api(mb_mgr); run++; @@ -1159,13 +1172,15 @@ direct_api_test(struct IMB_MGR *mb_mgr) errors += test_aes_api(mb_mgr); run++; +#endif /* __aarch64__ */ - errors += test_kasumi_api(mb_mgr); + errors += test_zuc_api(mb_mgr); run++; -#endif /* __x86_64__ */ - errors += test_zuc_api(mb_mgr); +#ifndef __aarch64__ + errors += test_kasumi_api(mb_mgr); run++; +#endif /* __aarch64__ */ errors += test_snow3g_api(mb_mgr); run++; @@ -1178,5 +1193,5 @@ direct_api_test(struct IMB_MGR *mb_mgr) #ifndef DEBUG signal(SIGSEGV, handler); #endif - return errors; + return errors; } diff --git a/test/do_test.h b/test/kat-app/do_test.h similarity index 98% rename from test/do_test.h rename to test/kat-app/do_test.h index f00c0df19fae8ec707d0315ec7fb6314ca482af5..2790eb7837a80c924541ec1d27eeb0dbfdb4c350 100644 --- a/test/do_test.h +++ b/test/kat-app/do_test.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2012-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -161,7 +161,8 @@ known_answer_test(IMB_MGR *mb_mgr) job->src = cipherCBC128; job->cipher_start_src_offset_in_bytes = 0; job->msg_len_to_cipher_in_bytes = NUMBYTES; - job->hash_start_src_offset_in_bytes = text - job->src; + job->hash_start_src_offset_in_bytes = + (uintptr_t) text - (uintptr_t) job->src; job->msg_len_to_hash_in_bytes = TEXTSIZE; job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; diff --git a/test/ecb_test.c b/test/kat-app/ecb_test.c similarity index 98% rename from test/ecb_test.c rename to test/kat-app/ecb_test.c index 58a032585eee498a6c197935c4089dffdb019e83..bf516026a8e1e5771666543abac104f8104552b9 100644 --- a/test/ecb_test.c +++ b/test/kat-app/ecb_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -716,17 +716,20 @@ test_ecb_vectors(struct IMB_MGR *mb_mgr, const int vec_cnt, DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); - printf("%s (N jobs = %d):\n", banner, num_jobs); + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); for (vect = 0; vect < vec_cnt; vect++) { struct test_suite_context *ctx = NULL; + if (!quiet_mode) { #ifdef DEBUG - printf("[%d/%d] Standard vector key_len:%d\n", - vect + 1, vec_cnt, - (int) vec_tab[vect].Klen); + printf("[%d/%d] Standard vector key_len:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].Klen); #else - printf("."); + printf("."); #endif + } switch (vec_tab[vect].Klen) { case 16: @@ -791,7 +794,8 @@ test_ecb_vectors(struct IMB_MGR *mb_mgr, const int vec_cnt, test_suite_update(ctx, 1, 0); } } - printf("\n"); + if (!quiet_mode) + printf("\n"); } int diff --git a/test/gcm_ctr_vectors_test.h b/test/kat-app/gcm_ctr_vectors_test.h similarity index 98% rename from test/gcm_ctr_vectors_test.h rename to test/kat-app/gcm_ctr_vectors_test.h index 36c8d9f197ba1a12e87d96183a4eb4d7b1bca3f9..3739618ea1905944f64b1639c6a3f624b4fb37bf 100644 --- a/test/gcm_ctr_vectors_test.h +++ b/test/kat-app/gcm_ctr_vectors_test.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2017-2022, Intel Corporation + Copyright (c) 2017-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/gcm_test.c b/test/kat-app/gcm_test.c similarity index 84% rename from test/gcm_test.c rename to test/kat-app/gcm_test.c index 01fcbda27f191377f8365fb3c4424628fe1d1149..2706df5fd2c3ed03d33e9e902c8709c667e97ccc 100644 --- a/test/gcm_test.c +++ b/test/kat-app/gcm_test.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2011-2022 Intel Corporation All rights reserved. + Copyright(c) 2011-2023 Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -814,71 +814,6 @@ static uint8_t T22[] = { 0x4b, 0x15, 0x3c, 0x8d, 0x48, 0xa1, 0x79, 0x30 }; -/* GHASH vectors */ -static uint8_t K23[] = { - 0xA1, 0xF6, 0x25, 0x8C, 0x87, 0x7D, 0x5F, 0xCD, - 0x89, 0x64, 0x48, 0x45, 0x38, 0xBF, 0xC9, 0x2C -}; - -static uint8_t P23[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F -}; - -static uint8_t T23[] = { - 0x9E, 0xE5, 0xA5, 0x1F, 0xBE, 0x28, 0xA1, 0x15, - 0x3E, 0xF1, 0x96, 0xF5, 0x0B, 0xBF, 0x03, 0xCA -}; - -static uint8_t K24[] = { - 0x1F, 0x0A, 0x6D, 0xCC, 0x67, 0xB1, 0x87, 0x22, - 0x98, 0x22, 0x77, 0x91, 0xDD, 0xA1, 0x9B, 0x6A -}; - -static uint8_t P24[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, -}; - -static uint8_t T24[] = { - 0xB5, 0x40, 0xDA, 0x44, 0xA3, 0x8C, 0x9C, 0x2B, - 0x95, 0x8E, 0x4B, 0x0B -}; - -static uint8_t K25[] = { - 0x1F, 0x0A, 0x6D, 0xCC, 0x67, 0xB1, 0x87, 0x22, - 0x98, 0x22, 0x77, 0x91, 0xDD, 0xA1, 0x9B, 0x6A -}; - -static uint8_t P25[] = { - 0x05 -}; - -static uint8_t T25[] = { - 0xE6, 0xCE, 0x47, 0xB5, 0xFB, 0xF2, 0xEF, 0x37, - 0x51, 0xF1, 0x57, 0x53, 0xAD, 0x56, 0x4F, 0xED -}; - -static uint8_t K33[] = { - 0x1f, 0x0f, 0x8a, 0x3a, 0xca, 0x64, 0x2e, 0xde, - 0xb1, 0xdf, 0x8a, 0x52, 0x9a, 0x29, 0x76, 0xee -}; -static uint8_t P33[] = { - 0x9b, 0xb5, 0x92, 0x9f, 0xa7, 0xaa, 0x83, 0xfd, - 0x0c, 0xd1, 0x83, 0x3a, 0x8e, 0xd5, 0x4d, 0xda, - 0x6a, 0xaf, 0xa1, 0xc7, 0xa1, 0x32, 0x3a, 0xd4, - 0x92, 0x9a, 0x2c, 0x83, 0xc6, 0x27, 0x92, 0x59, - 0x28, 0x90, 0x11, 0xde, 0x19, 0x4e, 0xd5, 0x16, - 0xef, 0x4f, 0x72, 0xeb, 0x79, 0x18, 0xd5, 0xb1, - 0xc5, 0x22, 0x40, 0x14, 0x92, 0xa2 -}; -static uint8_t T33[] = { - 0x8B, 0xA5, 0x3F, 0x5F, 0xD7, 0x0E, 0x55, 0x7C, - 0x30, 0xD4, 0xF2, 0xE1, 0x1A, 0x4F, 0xF8, 0xC7 -}; - /* GCM vectors with IV different than 12 bytes */ /* @@ -1123,138 +1058,6 @@ static uint8_t T34[] = { }; #define A34_len sizeof(A34) -/* - * GMAC vectors - */ -static uint8_t K29[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F -}; -static uint8_t IV29[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0A, 0x0B -}; -static uint8_t P29[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, -}; -static uint8_t T29[] = { - 0xC5, 0x3A, 0xF9, 0xE8 -}; - -#define C29 NULL -#define C29_len 0 -#define A29 NULL -#define A29_len 0 - -static uint8_t K30[] = { - 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, - 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 -}; -static uint8_t IV30[] = { - 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, - 0xde, 0xca, 0xf8, 0x88 -}; -static uint8_t P30[] = { - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10 -}; -static uint8_t T30[] = { - 0x4C, 0x0C, 0x4F, 0x47, 0x2D, 0x78, 0xF6, 0xD8, - 0x03, 0x53, 0x20, 0x2F, 0x1A, 0xDF, 0x90, 0xD0 -}; - -#define C30 NULL -#define C30_len 0 -#define A30 NULL -#define A30_len 0 - -static uint8_t K31[] = { - 0xaa, 0x74, 0x0a, 0xbf, 0xad, 0xcd, 0xa7, 0x79, - 0x22, 0x0d, 0x3b, 0x40, 0x6c, 0x5d, 0x7e, 0xc0, - 0x9a, 0x77, 0xfe, 0x9d, 0x94, 0x10, 0x45, 0x39, -}; -static uint8_t IV31[] = { - 0xab, 0x22, 0x65, 0xb4, 0xc1, 0x68, 0x95, - 0x55, 0x61, 0xf0, 0x43, 0x15 -}; -static uint8_t P31[] = { - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, -}; -static uint8_t T31[] = { - 0xCF, 0x82, 0x80, 0x64, 0x02, 0x46, 0xF4, 0xFB, - 0x33, 0xAE, 0x1D, 0x90, 0xEA, 0x48, 0x83, 0xDB -}; - -#define C31 NULL -#define C31_len 0 -#define A31 NULL -#define A31_len 0 - -static uint8_t K32[] = { - 0xb5, 0x48, 0xe4, 0x93, 0x4f, 0x5c, 0x64, 0xd3, - 0xc0, 0xf0, 0xb7, 0x8f, 0x7b, 0x4d, 0x88, 0x24, - 0xaa, 0xc4, 0x6b, 0x3c, 0x8d, 0x2c, 0xc3, 0x5e, - 0xe4, 0xbf, 0xb2, 0x54, 0xe4, 0xfc, 0xba, 0xf7, -}; -static uint8_t IV32[] = { - 0x2e, 0xed, 0xe1, 0xdc, 0x64, 0x47, 0xc7, - 0xaf, 0xc4, 0x41, 0x53, 0x58, -}; -static uint8_t P32[] = { - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, - 0x01 -}; -static uint8_t T32[] = { - 0x77, 0x46, 0x0D, 0x6F, 0xB1, 0x87, 0xDB, 0xA9, - 0x46, 0xAD, 0xCD, 0xFB, 0xB7, 0xF9, 0x13, 0xA1 -}; - -#define C32 NULL -#define C32_len 0 -#define A32 NULL -#define A32_len 0 - static const struct gcm_ctr_vector gcm_vectors[] = { /* * field order {K, Klen, IV, IVlen, A, Alen, P, Plen, C, T, Tlen}; @@ -1296,20 +1099,6 @@ static const struct gcm_ctr_vector gcm_iv_vectors[] = { vector(34) }; -static const struct gcm_ctr_vector ghash_vectors[] = { - ghash_vector(23), - ghash_vector(24), - ghash_vector(25), - ghash_vector(33) -}; - -static const struct gcm_ctr_vector gmac_vectors[] = { - vector(29), - vector(30), - vector(31), - vector(32), -}; - typedef int (*gcm_enc_dec_fn_t)(IMB_MGR *, const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint8_t *, uint64_t, @@ -1629,6 +1418,75 @@ sgl_aes_gcm_dec(IMB_MGR *p_mgr, return 0; } +/***************************************************************************** + * QUIC API + *****************************************************************************/ +static int +quic_aes_gcm_enc(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data *ctx, + uint8_t *out, const uint8_t *in, uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, uint64_t aad_len, + uint8_t *auth_tag, uint64_t auth_tag_len, + IMB_KEY_SIZE_BYTES key_len) +{ + if (iv_len != 12) + return aes_gcm_enc(p_mgr, key, ctx, + out, in, len, + iv, iv_len, + aad, aad_len, + auth_tag, auth_tag_len, key_len); + + imb_quic_aes_gcm(p_mgr, key, key_len, IMB_DIR_ENCRYPT, + (void **) &out, (const void * const*) &in, &len, + (const void * const*) &iv, + (const void * const*) &aad, aad_len, + (void **) &auth_tag, auth_tag_len, 1); + + const int err = imb_get_errno(p_mgr); + + if (err != 0) { + printf("QUIC GCM encrypt error %d, %s\n", + err, imb_get_strerror(err)); + return 1; + } + return 0; +} + +static int +quic_aes_gcm_dec(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data *ctx, + uint8_t *out, const uint8_t *in, uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, uint64_t aad_len, + uint8_t *auth_tag, uint64_t auth_tag_len, + IMB_KEY_SIZE_BYTES key_len) +{ + if (iv_len != 12) + return aes_gcm_dec(p_mgr, key, ctx, + out, in, len, + iv, iv_len, + aad, aad_len, + auth_tag, auth_tag_len, key_len); + + imb_quic_aes_gcm(p_mgr, key, key_len, IMB_DIR_DECRYPT, + (void **) &out, (const void * const*) &in, &len, + (const void * const*) &iv, + (const void * const*) &aad, aad_len, + (void **) &auth_tag, auth_tag_len, 1); + + const int err = imb_get_errno(p_mgr); + + if (err != 0) { + printf("QUIC GCM decrypt error %d, %s\n", + err, imb_get_strerror(err)); + return 1; + } + return 0; +} + /***************************************************************************** * burst API *****************************************************************************/ @@ -1637,10 +1495,10 @@ aes_gcm_burst(IMB_MGR *mb_mgr, const IMB_CIPHER_DIRECTION cipher_dir, const struct gcm_key_data *key, const uint64_t key_len, - uint8_t **out, const uint8_t *in, const uint64_t len, + uint8_t ** const out, const uint8_t *in, const uint64_t len, const uint8_t *iv, const uint64_t iv_len, const uint8_t *aad, - const uint64_t aad_len, uint8_t **auth_tag, - const uint64_t auth_tag_len, struct gcm_context_data **ctx, + const uint64_t aad_len, uint8_t ** const auth_tag, + const uint64_t auth_tag_len, struct gcm_context_data ** const ctx, const IMB_CIPHER_MODE cipher_mode, const IMB_SGL_STATE sgl_state, const uint32_t num_jobs) { @@ -1678,6 +1536,8 @@ aes_gcm_burst(IMB_MGR *mb_mgr, job->hash_alg = IMB_AUTH_GCM_SGL; } else job->hash_alg = IMB_AUTH_AES_GMAC; + + imb_set_session(mb_mgr, job); } const uint32_t completed_jobs = @@ -1725,6 +1585,51 @@ burst_aes_gcm_enc(IMB_MGR *p_mgr, num_jobs); } +static int +burst_quic_gcm_enc(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data **ctx, uint8_t **out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t **auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len, const uint32_t num_jobs) +{ + if (iv_len != 12) { + return burst_aes_gcm_enc(p_mgr, key, ctx, out, in, len, + iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, + key_len, num_jobs); + } + + const void *in_array[GCM_MAX_JOBS]; + uint64_t len_array[GCM_MAX_JOBS]; + const void *iv_array[GCM_MAX_JOBS]; + const void *aad_array[GCM_MAX_JOBS]; + uint32_t i; + + for (i = 0; i < num_jobs; i++) { + in_array[i] = (const void *) in; + len_array[i] = len; + iv_array[i] = (const void *) iv; + aad_array[i] = (const void *) aad; + } + + imb_quic_aes_gcm(p_mgr, key, key_len, IMB_DIR_ENCRYPT, + (void **) out, in_array, len_array, + iv_array, aad_array, aad_len, + (void **) auth_tag, auth_tag_len, num_jobs); + + const int err = imb_get_errno(p_mgr); + + if (err != 0) { + printf("QUIC GCM burst-encrypt error %d, %s\n", + err, imb_get_strerror(err)); + return 1; + } + return 0; +} + static int burst_aes_gcm_dec(IMB_MGR *p_mgr, const struct gcm_key_data *key, @@ -1741,6 +1646,51 @@ burst_aes_gcm_dec(IMB_MGR *p_mgr, num_jobs); } +static int +burst_quic_gcm_dec(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data **ctx, uint8_t **out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t **auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len, const uint32_t num_jobs) +{ + if (iv_len != 12) { + return burst_aes_gcm_dec(p_mgr, key, ctx, out, in, len, + iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, + key_len, num_jobs); + } + + const void *in_array[GCM_MAX_JOBS]; + uint64_t len_array[GCM_MAX_JOBS]; + const void *iv_array[GCM_MAX_JOBS]; + const void *aad_array[GCM_MAX_JOBS]; + uint32_t i; + + for (i = 0; i < num_jobs; i++) { + in_array[i] = (const void *) in; + len_array[i] = len; + iv_array[i] = (const void *) iv; + aad_array[i] = (const void *) aad; + } + + imb_quic_aes_gcm(p_mgr, key, key_len, IMB_DIR_DECRYPT, + (void **) out, in_array, len_array, + iv_array, aad_array, aad_len, + (void **) auth_tag, auth_tag_len, num_jobs); + + const int err = imb_get_errno(p_mgr); + + if (err != 0) { + printf("QUIC GCM burst-decrypt error %d, %s\n", + err, imb_get_strerror(err)); + return 1; + } + return 0; +} + static int burst_sgl_aes_gcm(IMB_MGR *p_mgr, IMB_CIPHER_DIRECTION cipher_dir, @@ -2145,6 +2095,7 @@ test_gcm_vectors(struct gcm_ctr_vector const *vector, free(T2_test); } + static void test_gcm_vectors_burst(struct gcm_ctr_vector const *vector, gcm_enc_dec_many_fn_t encfn, @@ -2318,18 +2269,21 @@ test_gcm_std_vectors(struct test_suite_context *ts128, printf("AES-GCM (%s API) standard test vectors:\n", test_sgl_api ? "SGL" : "Direct/JOB"); for (vect = 0; vect < vectors_cnt; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("Standard vector %d/%d Keylen:%d IVlen:%d PTLen:%d " - "AADlen:%d Tlen:%d\n", - vect + 1, vectors_cnt, - (int) vectors[vect].Klen, - (int) vectors[vect].IVlen, - (int) vectors[vect].Plen, - (int) vectors[vect].Alen, - (int) vectors[vect].Tlen); + printf("Standard vector %d/%d Keylen:%d IVlen:%d " + "PTLen:%d AADlen:%d Tlen:%d\n", + vect + 1, vectors_cnt, + (int) vectors[vect].Klen, + (int) vectors[vect].IVlen, + (int) vectors[vect].Plen, + (int) vectors[vect].Alen, + (int) vectors[vect].Tlen); #else - printf("."); + printf("."); #endif + } + switch (vectors[vect].Klen) { case IMB_KEY_128_BYTES: if (test_sgl_api) { @@ -2358,6 +2312,14 @@ test_gcm_std_vectors(struct test_suite_context *ts128, burst_aes_gcm_enc, burst_aes_gcm_dec, ts128); + test_gcm_vectors(&vectors[vect], + quic_aes_gcm_enc, + quic_aes_gcm_dec, + ts128); + test_gcm_vectors_burst(&vectors[vect], + burst_quic_gcm_enc, + burst_quic_gcm_dec, + ts128); } break; case IMB_KEY_192_BYTES: @@ -2387,6 +2349,7 @@ test_gcm_std_vectors(struct test_suite_context *ts128, burst_aes_gcm_enc, burst_aes_gcm_dec, ts192); + /* AES-192 is not supported by QUIC */ } break; case IMB_KEY_256_BYTES: @@ -2417,7 +2380,14 @@ test_gcm_std_vectors(struct test_suite_context *ts128, burst_aes_gcm_enc, burst_aes_gcm_dec, ts256); - + test_gcm_vectors(&vectors[vect], + quic_aes_gcm_enc, + quic_aes_gcm_dec, + ts256); + test_gcm_vectors_burst(&vectors[vect], + burst_quic_gcm_enc, + burst_quic_gcm_dec, + ts256); } break; default: @@ -2425,253 +2395,8 @@ test_gcm_std_vectors(struct test_suite_context *ts128, return; } } - printf("\n"); -} - -static void -test_ghash(struct test_suite_context *ts, const int use_job_api) -{ - const int vectors_cnt = DIM(ghash_vectors); - int vect; - - printf("GHASH test vectors (%s API):\n", - use_job_api ? "job" : "direct"); - for (vect = 0; vect < vectors_cnt; vect++) { - struct gcm_key_data gdata_key; - struct gcm_ctr_vector const *vector = &ghash_vectors[vect]; - uint8_t T_test[16]; - - memset(&gdata_key, 0, sizeof(struct gcm_key_data)); - memset(T_test, 0, sizeof(T_test)); - IMB_GHASH_PRE(p_gcm_mgr, vector->K, &gdata_key); - - if (!use_job_api) { - IMB_GHASH(p_gcm_mgr, &gdata_key, vector->P, - vector->Plen, T_test, vector->Tlen); - } else { - IMB_JOB *job = IMB_GET_NEXT_JOB(p_gcm_mgr); - - if (!job) { - fprintf(stderr, - "failed to get job for ghash\n"); - return; - } - - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_GHASH; - job->u.GHASH._key = &gdata_key; - job->u.GHASH._init_tag = T_test; - job->src = vector->P; - job->msg_len_to_hash_in_bytes = vector->Plen; - job->hash_start_src_offset_in_bytes = UINT64_C(0); - job->auth_tag_output = T_test; - job->auth_tag_output_len_in_bytes = vector->Tlen; - - job = IMB_SUBMIT_JOB(p_gcm_mgr); - while (job) { - if (job->status != IMB_STATUS_COMPLETED) - fprintf(stderr, - "failed job, status:%d\n", - job->status); - job = IMB_GET_COMPLETED_JOB(p_gcm_mgr); - } - while ((job = IMB_FLUSH_JOB(p_gcm_mgr)) != NULL) { - if (job->status != IMB_STATUS_COMPLETED) - fprintf(stderr, - "failed job, status:%d\n", - job->status); - } - } - - if (check_data(T_test, vector->T, vector->Tlen, - "generated tag (T)")) - test_suite_update(ts, 0, 1); - else - test_suite_update(ts, 1, 0); - } -} - -static void -aes_gmac_job(IMB_MGR *mb_mgr, - const uint8_t *k, - struct gcm_key_data *gmac_key, - const uint64_t key_len, - const uint8_t *in, const uint64_t len, - const uint8_t *iv, const uint64_t iv_len, - uint8_t *auth_tag, const uint64_t auth_tag_len) -{ - IMB_JOB *job; - - job = IMB_GET_NEXT_JOB(mb_mgr); - if (!job) { - fprintf(stderr, "failed to get job\n"); - return; - } - - if (key_len == 16) { - IMB_AES128_GCM_PRE(mb_mgr, k, gmac_key); - job->hash_alg = IMB_AUTH_AES_GMAC_128; - } else if (key_len == 24) { - IMB_AES192_GCM_PRE(mb_mgr, k, gmac_key); - job->hash_alg = IMB_AUTH_AES_GMAC_192; - } else { /* key_len == 32 */ - IMB_AES256_GCM_PRE(mb_mgr, k, gmac_key); - job->hash_alg = IMB_AUTH_AES_GMAC_256; - } - - job->cipher_mode = IMB_CIPHER_NULL; - job->u.GMAC._key = gmac_key; - job->u.GMAC._iv = iv; - job->u.GMAC.iv_len_in_bytes = iv_len; - job->src = in; - job->msg_len_to_hash_in_bytes = len; - job->hash_start_src_offset_in_bytes = UINT64_C(0); - job->auth_tag_output = auth_tag; - job->auth_tag_output_len_in_bytes = auth_tag_len; - - job = IMB_SUBMIT_JOB(mb_mgr); - while (job) { - if (job->status != IMB_STATUS_COMPLETED) - fprintf(stderr, "failed job, status:%d\n", job->status); - job = IMB_GET_COMPLETED_JOB(mb_mgr); - } - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - if (job->status != IMB_STATUS_COMPLETED) - fprintf(stderr, "failed job, status:%d\n", job->status); - } -} - -#define MAX_SEG_SIZE 64 -static void -test_gmac_vector(const struct gcm_ctr_vector *vector, - const uint64_t seg_size, - const unsigned job_api, - struct test_suite_context *ts128, - struct test_suite_context *ts192, - struct test_suite_context *ts256) -{ - struct gcm_key_data key; - struct gcm_context_data ctx; - const uint8_t *iv = vector->IV; - const uint64_t iv_len = vector->IVlen; - const uint64_t nb_segs = (vector->Plen / seg_size); - const uint64_t last_partial_seg = (vector->Plen % seg_size); - uint8_t in_seg[MAX_SEG_SIZE]; - const uint8_t *in_ptr = vector->P; - uint32_t i; - uint8_t T_test[16]; - struct test_suite_context *ts = ts128; - - if (vector->Klen == IMB_KEY_192_BYTES) - ts = ts192; - - if (vector->Klen == IMB_KEY_256_BYTES) - ts = ts256; - - memset(&key, 0, sizeof(struct gcm_key_data)); - if (job_api) - aes_gmac_job(p_gcm_mgr, vector->K, &key, vector->Klen, in_ptr, - seg_size, iv, iv_len, T_test, vector->Tlen); - else { - switch (vector->Klen) { - case IMB_KEY_128_BYTES: - IMB_AES128_GCM_PRE(p_gcm_mgr, vector->K, &key); - IMB_AES128_GMAC_INIT(p_gcm_mgr, &key, &ctx, iv, iv_len); - in_ptr = vector->P; - for (i = 0; i < nb_segs; i++) { - memcpy(in_seg, in_ptr, seg_size); - IMB_AES128_GMAC_UPDATE(p_gcm_mgr, &key, &ctx, - in_seg, - seg_size); - in_ptr += seg_size; - } - - if (last_partial_seg != 0) { - memcpy(in_seg, in_ptr, last_partial_seg); - IMB_AES128_GMAC_UPDATE(p_gcm_mgr, &key, &ctx, - in_seg, - last_partial_seg); - } - - IMB_AES128_GMAC_FINALIZE(p_gcm_mgr, &key, &ctx, T_test, - vector->Tlen); - break; - case IMB_KEY_192_BYTES: - IMB_AES192_GCM_PRE(p_gcm_mgr, vector->K, &key); - IMB_AES192_GMAC_INIT(p_gcm_mgr, &key, &ctx, iv, iv_len); - in_ptr = vector->P; - for (i = 0; i < nb_segs; i++) { - memcpy(in_seg, in_ptr, seg_size); - IMB_AES192_GMAC_UPDATE(p_gcm_mgr, &key, &ctx, - in_seg, - seg_size); - in_ptr += seg_size; - } - - if (last_partial_seg != 0) { - memcpy(in_seg, in_ptr, last_partial_seg); - IMB_AES192_GMAC_UPDATE(p_gcm_mgr, &key, &ctx, - in_seg, - last_partial_seg); - } - - IMB_AES192_GMAC_FINALIZE(p_gcm_mgr, &key, &ctx, T_test, - vector->Tlen); - break; - case IMB_KEY_256_BYTES: - default: - IMB_AES256_GCM_PRE(p_gcm_mgr, vector->K, &key); - IMB_AES256_GMAC_INIT(p_gcm_mgr, &key, &ctx, iv, iv_len); - in_ptr = vector->P; - for (i = 0; i < nb_segs; i++) { - memcpy(in_seg, in_ptr, seg_size); - IMB_AES256_GMAC_UPDATE(p_gcm_mgr, &key, &ctx, - in_seg, - seg_size); - in_ptr += seg_size; - } - - if (last_partial_seg != 0) { - memcpy(in_seg, in_ptr, last_partial_seg); - IMB_AES256_GMAC_UPDATE(p_gcm_mgr, &key, &ctx, - in_seg, - last_partial_seg); - } - - IMB_AES256_GMAC_FINALIZE(p_gcm_mgr, &key, &ctx, T_test, - vector->Tlen); - break; - } - } - - if (check_data(T_test, vector->T, vector->Tlen, "generated tag (T)")) - test_suite_update(ts, 0, 1); - else - test_suite_update(ts, 1, 0); -} - -static void -test_gmac(struct test_suite_context *ts128, - struct test_suite_context *ts192, - struct test_suite_context *ts256) -{ - const int vectors_cnt = DIM(gmac_vectors); - int vect; - - printf("GMAC test vectors:\n"); - for (vect = 0; vect < vectors_cnt; vect++) { - const struct gcm_ctr_vector *vector = &gmac_vectors[vect]; - uint64_t seg_size; - - /* Using direct API, which allows SGL */ - for (seg_size = 1; seg_size <= MAX_SEG_SIZE; seg_size++) - test_gmac_vector(vector, seg_size, 0, - ts128, ts192, ts256); - - /* Using job API */ - test_gmac_vector(vector, vector->Plen, 1, - ts128, ts192, ts256); - } + if (!quiet_mode) + printf("\n"); } static void @@ -3128,18 +2853,5 @@ int gcm_test(IMB_MGR *p_mgr) errors += test_suite_end(&ts192); errors += test_suite_end(&ts256); - test_suite_start(&ts128, "AES-GMAC-128"); - test_suite_start(&ts192, "AES-GMAC-192"); - test_suite_start(&ts256, "AES-GMAC-256"); - test_gmac(&ts128, &ts192, &ts256); - errors += test_suite_end(&ts128); - errors += test_suite_end(&ts192); - errors += test_suite_end(&ts256); - - test_suite_start(&ts128, "GHASH"); - test_ghash(&ts128, 0); - test_ghash(&ts128, 1); - errors += test_suite_end(&ts128); - return errors; } diff --git a/test/kat-app/ghash_test.c b/test/kat-app/ghash_test.c new file mode 100644 index 0000000000000000000000000000000000000000..1220f0ab26597cfe3fc50e64740a4a6994f015dc --- /dev/null +++ b/test/kat-app/ghash_test.c @@ -0,0 +1,132 @@ +/********************************************************************** + Copyright(c) 2023 Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include +#include +#include +#include /* for memcmp() */ + +#include +#include "utils.h" +#include "mac_test.h" + +int ghash_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test ghash_test_json[]; + +static int check_data(const uint8_t *test, const char *expected, + uint64_t len, const char *data_name) +{ + int mismatch; + int is_error = 0; + + if (len == 0) + return is_error; + + if (test == NULL || expected == NULL || data_name == NULL) + return 1; + + mismatch = memcmp(test, expected, len); + if (mismatch) { + uint64_t a; + + is_error = 1; + printf(" expected results don't match %s \t\t", data_name); + for (a = 0; a < len; a++) + if (test[a] != expected[a]) { + printf(" '%x' != '%x' at %llx of %llx\n", test[a], expected[a], + (unsigned long long) a, (unsigned long long) len); + break; + } + } + return is_error; +} + +int ghash_test(struct IMB_MGR *mb_mgr) +{ + struct test_suite_context ts; + int use_job_api = 0; + + test_suite_start(&ts, "GHASH"); + + while (use_job_api < 2) { + const struct mac_test *vec = ghash_test_json; + + printf("GHASH test vectors (%s API):\n", use_job_api ? "job" : "direct"); + while (vec->msg != NULL) { + struct gcm_key_data gdata_key; + uint8_t T_test[16]; + + memset(&gdata_key, 0, sizeof(struct gcm_key_data)); + memset(T_test, 0, sizeof(T_test)); + IMB_GHASH_PRE(mb_mgr, vec->key, &gdata_key); + + if (!use_job_api) { + IMB_GHASH(mb_mgr, &gdata_key, vec->msg, + (vec->msgSize / 8), T_test, vec->tagSize); + } else { + IMB_JOB *job = IMB_GET_NEXT_JOB(mb_mgr); + + if (!job) { + fprintf(stderr, "failed to get job for ghash\n"); + test_suite_update(&ts, 0, 1); + return test_suite_end(&ts); + } + + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_GHASH; + job->u.GHASH._key = &gdata_key; + job->u.GHASH._init_tag = T_test; + job->src = (const void *) vec->msg; + job->msg_len_to_hash_in_bytes = (vec->msgSize / 8); + job->hash_start_src_offset_in_bytes = UINT64_C(0); + job->auth_tag_output = T_test; + job->auth_tag_output_len_in_bytes = vec->tagSize; + + job = IMB_SUBMIT_JOB(mb_mgr); + + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job == NULL) + fprintf(stderr, "No job retrieved\n"); + else if (job->status != IMB_STATUS_COMPLETED) + fprintf(stderr, "failed job, status:%d\n", job->status); + } + + if (check_data(T_test, vec->tag, vec->tagSize, "generated tag (T)")) + test_suite_update(&ts, 0, 1); + else + test_suite_update(&ts, 1, 0); + vec++; + } + use_job_api++; + } + + return test_suite_end(&ts); +} diff --git a/test/gcm_vectors.h b/test/kat-app/ghash_test.json.c similarity index 54% rename from test/gcm_vectors.h rename to test/kat-app/ghash_test.json.c index 71eb41707c95e3586b132d6aebc669b964c43e5f..bfaf87f90c09db4d2fe5a08ecd888e4292b7ce6d 100644 --- a/test/gcm_vectors.h +++ b/test/kat-app/ghash_test.json.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2011-2022 Intel Corporation All rights reserved. + Copyright(c) 2023 Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -27,12 +27,30 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ -#ifndef AES_GCM_VECTORS_H_ -#define AES_GCM_VECTORS_H_ +/* GHASH */ +#include "mac_test.h" -#include - -#include "gcm_std_vectors_test.h" - - -#endif /* AES_GCM_VECTORS_H_ */ +const struct mac_test ghash_test_json[] = { + {16, 16, 1, + "\xa1\xf6\x25\x8c\x87\x7d\x5f\xcd\x89\x64\x48\x45\x38\xbf\xc9\x2c", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x9e\xe5\xa5\x1f\xbe\x28\xa1\x15\x3e\xf1\x96\xf5\x0b\xbf\x03\xca", 1, 128, + NULL, 0}, + {16, 12, 2, + "\x1f\x0a\x6d\xcc\x67\xb1\x87\x22\x98\x22\x77\x91\xdd\xa1\x9b\x6a", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\xb5\x40\xda\x44\xa3\x8c\x9c\x2b\x95\x8e\x4b\x0b", 1, 256, NULL, 0}, + {16, 16, 3, + "\x1f\x0a\x6d\xcc\x67\xb1\x87\x22\x98\x22\x77\x91\xdd\xa1\x9b\x6a", "\x05", + "\xe6\xce\x47\xb5\xfb\xf2\xef\x37\x51\xf1\x57\x53\xad\x56\x4f\xed", 1, 8, + NULL, 0}, + {16, 16, 4, + "\x1f\x0f\x8a\x3a\xca\x64\x2e\xde\xb1\xdf\x8a\x52\x9a\x29\x76\xee", + "\x9b\xb5\x92\x9f\xa7\xaa\x83\xfd\x0c\xd1\x83\x3a\x8e\xd5\x4d\xda\x6a\xaf" + "\xa1\xc7\xa1\x32\x3a\xd4\x92\x9a\x2c\x83\xc6\x27\x92\x59\x28\x90\x11\xde" + "\x19\x4e\xd5\x16\xef\x4f\x72\xeb\x79\x18\xd5\xb1\xc5\x22\x40\x14\x92\xa2", + "\x8b\xa5\x3f\x5f\xd7\x0e\x55\x7c\x30\xd4\xf2\xe1\x1a\x4f\xf8\xc7", 1, 432, + NULL, 0}, + {0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0} +}; diff --git a/test/kat-app/gmac_test.c b/test/kat-app/gmac_test.c new file mode 100644 index 0000000000000000000000000000000000000000..c582f25c81b00407c536b337d3e74aba1629818d --- /dev/null +++ b/test/kat-app/gmac_test.c @@ -0,0 +1,261 @@ +/********************************************************************** + Copyright(c) 2023 Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include +#include +#include +#include /* for memcmp() */ + +#include +#include "utils.h" +#include "mac_test.h" + +int gmac_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test gmac_test_kat_json[]; + +static int check_data(const uint8_t *test, const uint8_t *expected, + uint64_t len, const char *data_name) +{ + int mismatch; + int is_error = 0; + + if (len == 0) + return is_error; + + if (test == NULL || expected == NULL || data_name == NULL) + return 1; + + mismatch = memcmp(test, expected, len); + if (mismatch) { + uint64_t a; + + is_error = 1; + printf(" expected results don't match %s \t\t", data_name); + for (a = 0; a < len; a++) { + if (test[a] != expected[a]) { + printf(" '%x' != '%x' at %llx of %llx\n", + test[a], expected[a], + (unsigned long long) a, + (unsigned long long) len); + break; + } + } + } + return is_error; +} + +static void +aes_gmac_job(IMB_MGR *mb_mgr, + const uint8_t *k, + struct gcm_key_data *gmac_key, + const uint64_t key_len, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + uint8_t *auth_tag, const uint64_t auth_tag_len) +{ + IMB_JOB *job; + + job = IMB_GET_NEXT_JOB(mb_mgr); + if (!job) { + fprintf(stderr, "failed to get job\n"); + return; + } + + if (key_len == 16) { + IMB_AES128_GCM_PRE(mb_mgr, k, gmac_key); + job->hash_alg = IMB_AUTH_AES_GMAC_128; + } else if (key_len == 24) { + IMB_AES192_GCM_PRE(mb_mgr, k, gmac_key); + job->hash_alg = IMB_AUTH_AES_GMAC_192; + } else { /* key_len == 32 */ + IMB_AES256_GCM_PRE(mb_mgr, k, gmac_key); + job->hash_alg = IMB_AUTH_AES_GMAC_256; + } + + job->cipher_mode = IMB_CIPHER_NULL; + job->u.GMAC._key = gmac_key; + job->u.GMAC._iv = iv; + job->u.GMAC.iv_len_in_bytes = iv_len; + job->src = in; + job->msg_len_to_hash_in_bytes = len; + job->hash_start_src_offset_in_bytes = UINT64_C(0); + job->auth_tag_output = auth_tag; + job->auth_tag_output_len_in_bytes = auth_tag_len; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job == NULL) + fprintf(stderr, "No job retrieved\n"); + else if (job->status != IMB_STATUS_COMPLETED) + fprintf(stderr, "failed job, status:%d\n", job->status); +} + +#define MAX_SEG_SIZE 64 +static void +gmac_test_vector(IMB_MGR *mb_mgr, + const struct mac_test *vector, + const uint64_t seg_size, + const unsigned job_api, + struct test_suite_context *ts128, + struct test_suite_context *ts192, + struct test_suite_context *ts256) +{ + struct gcm_key_data key; + struct gcm_context_data ctx; + const uint8_t *iv = (const void *) vector->iv; + const uint64_t iv_len = vector->ivSize; + const uint64_t nb_segs = ((vector->msgSize / 8) / seg_size); + const uint64_t last_partial_seg = ((vector->msgSize / 8) % seg_size); + const uint8_t *in_ptr = (const void *) vector->msg; + uint8_t T_test[16]; + struct test_suite_context *ts = ts128; + + if (vector->keySize == IMB_KEY_192_BYTES) + ts = ts192; + + if (vector->keySize == IMB_KEY_256_BYTES) + ts = ts256; + + memset(&key, 0, sizeof(struct gcm_key_data)); + if (job_api) { + aes_gmac_job(mb_mgr, (const void *) vector->key, &key, vector->keySize, in_ptr, + seg_size, iv, iv_len, T_test, vector->tagSize); + } else { + uint8_t in_seg[MAX_SEG_SIZE]; + uint32_t i; + + switch (vector->keySize) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_PRE(mb_mgr, vector->key, &key); + IMB_AES128_GMAC_INIT(mb_mgr, &key, &ctx, iv, iv_len); + in_ptr = (const void *) vector->msg; + for (i = 0; i < nb_segs; i++) { + memcpy(in_seg, in_ptr, seg_size); + IMB_AES128_GMAC_UPDATE(mb_mgr, &key, &ctx, + in_seg, + seg_size); + in_ptr += seg_size; + } + + if (last_partial_seg != 0) { + memcpy(in_seg, in_ptr, last_partial_seg); + IMB_AES128_GMAC_UPDATE(mb_mgr, &key, &ctx, + in_seg, + last_partial_seg); + } + + IMB_AES128_GMAC_FINALIZE(mb_mgr, &key, &ctx, T_test, + vector->tagSize); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_PRE(mb_mgr, vector->key, &key); + IMB_AES192_GMAC_INIT(mb_mgr, &key, &ctx, iv, iv_len); + in_ptr = (const void *) vector->msg; + for (i = 0; i < nb_segs; i++) { + memcpy(in_seg, in_ptr, seg_size); + IMB_AES192_GMAC_UPDATE(mb_mgr, &key, &ctx, + in_seg, + seg_size); + in_ptr += seg_size; + } + + if (last_partial_seg != 0) { + memcpy(in_seg, in_ptr, last_partial_seg); + IMB_AES192_GMAC_UPDATE(mb_mgr, &key, &ctx, + in_seg, + last_partial_seg); + } + + IMB_AES192_GMAC_FINALIZE(mb_mgr, &key, &ctx, T_test, + vector->tagSize); + break; + case IMB_KEY_256_BYTES: + default: + IMB_AES256_GCM_PRE(mb_mgr, vector->key, &key); + IMB_AES256_GMAC_INIT(mb_mgr, &key, &ctx, iv, iv_len); + in_ptr = (const void *) vector->msg; + for (i = 0; i < nb_segs; i++) { + memcpy(in_seg, in_ptr, seg_size); + IMB_AES256_GMAC_UPDATE(mb_mgr, &key, &ctx, + in_seg, + seg_size); + in_ptr += seg_size; + } + + if (last_partial_seg != 0) { + memcpy(in_seg, in_ptr, last_partial_seg); + IMB_AES256_GMAC_UPDATE(mb_mgr, &key, &ctx, + in_seg, + last_partial_seg); + } + + IMB_AES256_GMAC_FINALIZE(mb_mgr, &key, &ctx, T_test, + vector->tagSize); + break; + } + } + + if (check_data(T_test, (const void *) vector->tag, vector->tagSize, "generated tag (T)")) + test_suite_update(ts, 0, 1); + else + test_suite_update(ts, 1, 0); +} + +int gmac_test(IMB_MGR *mb_mgr) +{ + struct test_suite_context ts128, ts192, ts256; + int errors = 0; + + test_suite_start(&ts128, "AES-GMAC-128"); + test_suite_start(&ts192, "AES-GMAC-192"); + test_suite_start(&ts256, "AES-GMAC-256"); + + printf("GMAC test vectors:\n"); + const struct mac_test *vec = gmac_test_kat_json; + while (vec->msg != NULL) { + uint64_t seg_size; + + /* Using direct API, which allows SGL */ + for (seg_size = 1; seg_size <= MAX_SEG_SIZE; seg_size++) + gmac_test_vector(mb_mgr, vec, seg_size, 0, + &ts128, &ts192, &ts256); + + /* Using job API */ + gmac_test_vector(mb_mgr, vec, (vec->msgSize / 8), 1, + &ts128, &ts192, &ts256); + vec++; + } + errors += test_suite_end(&ts128); + errors += test_suite_end(&ts192); + errors += test_suite_end(&ts256); + + return errors; +} diff --git a/test/kat-app/gmac_test.json.c b/test/kat-app/gmac_test.json.c new file mode 100644 index 0000000000000000000000000000000000000000..921f086e91ab3936f4f4207a64abcc1dbd271854 --- /dev/null +++ b/test/kat-app/gmac_test.json.c @@ -0,0 +1,78 @@ +/********************************************************************** + Copyright(c) 2023 Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +/* GMAC */ +#include "mac_test.h" + +const struct mac_test gmac_test_kat_json[] = { + /* Vectors from + http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-revised-spec.pdf + */ + {16, 4, 1, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23" + "\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35" + "\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\xc5\x3a\xf9\xe8", 1, 512, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 12}, + {16, 16, 2, + "\xfe\xff\xe9\x92\x86\x65\x73\x1c\x6d\x6a\x8f\x94\x67\x30\x83\x08", + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02" + "\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04" + "\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06" + "\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06\x07\x08" + "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a" + "\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c" + "\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + "\x0f\x10\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10", + "\x4c\x0c\x4f\x47\x2d\x78\xf6\xd8\x03\x53\x20\x2f\x1a\xdf\x90\xd0", 1, + 1280, "\xca\xfe\xba\xbe\xfa\xce\xdb\xad\xde\xca\xf8\x88", 12}, + {24, 16, 3, + "\xaa\x74\x0a\xbf\xad\xcd\xa7\x79\x22\x0d\x3b\x40\x6c\x5d\x7e\xc0\x9a\x77" + "\xfe\x9d\x94\x10\x45\x39", + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02" + "\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04" + "\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06" + "\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06\x07\x08" + "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10", + "\xcf\x82\x80\x64\x02\x46\xf4\xfb\x33\xae\x1d\x90\xea\x48\x83\xdb", 1, 640, + "\xab\x22\x65\xb4\xc1\x68\x95\x55\x61\xf0\x43\x15", 12}, + {32, 16, 4, + "\xb5\x48\xe4\x93\x4f\x5c\x64\xd3\xc0\xf0\xb7\x8f\x7b\x4d\x88\x24\xaa\xc4" + "\x6b\x3c\x8d\x2c\xc3\x5e\xe4\xbf\xb2\x54\xe4\xfc\xba\xf7", + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02" + "\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04" + "\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01\x02\x03\x04\x05\x06" + "\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x01", + "\x77\x46\x0d\x6f\xb1\x87\xdb\xa9\x46\xad\xcd\xfb\xb7\xf9\x13\xa1", 1, 520, + "\x2e\xed\xe1\xdc\x64\x47\xc7\xaf\xc4\x41\x53\x58", 12}, + {0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0} +}; diff --git a/test/hec_test.c b/test/kat-app/hec_test.c similarity index 89% rename from test/hec_test.c rename to test/kat-app/hec_test.c index 144ca180a1b36f7836dccaa2fa78d4b6ebed75c0..d6d58681e511c101beb19ad44c78c0db230c3ca3 100644 --- a/test/hec_test.c +++ b/test/kat-app/hec_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -58,8 +58,9 @@ test_32_bit(IMB_MGR *mgr, struct test_suite_context *ctx) uint32_t out = 0; const uint8_t *in_p = (const uint8_t *) ∈ #ifdef DEBUG - printf("[32-bit %u] PF | HEC:\t0x%08lx", i + 1, - (unsigned long) expected_out); + if (!quiet_mode) + printf("[32-bit %u] PF | HEC:\t0x%08lx", i + 1, + (unsigned long) expected_out); #endif out = IMB_HEC_32(mgr, in_p); @@ -70,7 +71,8 @@ test_32_bit(IMB_MGR *mgr, struct test_suite_context *ctx) } else { test_suite_update(ctx, 1, 0); #ifdef DEBUG - printf("\tHEC 32 - Pass\n"); + if (!quiet_mode) + printf("\tHEC 32 - Pass\n"); #endif } } @@ -100,8 +102,9 @@ test_64_bit(IMB_MGR *mgr, struct test_suite_context *ctx) uint64_t out = 0; const uint8_t *in_p = (const uint8_t *) ∈ #ifdef DEBUG - printf("[64-bit %u] PF | HEC:\t0x%016llx", i + 1, - (unsigned long long)expected_out); + if (!quiet_mode) + printf("[64-bit %u] PF | HEC:\t0x%016llx", i + 1, + (unsigned long long)expected_out); #endif out = IMB_HEC_64(mgr, in_p); @@ -112,7 +115,8 @@ test_64_bit(IMB_MGR *mgr, struct test_suite_context *ctx) } else { test_suite_update(ctx, 1, 0); #ifdef DEBUG - printf("\tHEC 64 - Pass\n"); + if (!quiet_mode) + printf("\tHEC 64 - Pass\n"); #endif } } diff --git a/test/kat-app/hmac_md5.json.c b/test/kat-app/hmac_md5.json.c new file mode 100644 index 0000000000000000000000000000000000000000..a61daa574467348439fa8858747c786c00a892ff --- /dev/null +++ b/test/kat-app/hmac_md5.json.c @@ -0,0 +1,137 @@ +/******************************************************************************* + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* HMACMD5, RFC2202 */ +#include "mac_test.h" + +const struct mac_test hmac_md5_test_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc2202 */ + {16, 16, 1, + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", + "\x48\x69\x20\x54\x68\x65\x72\x65", + "\x92\x94\x72\x7a\x36\x38\xbb\x1c\x13\xf4\x8e\xf8\x15\x8b\xfc\x9d", 1, 64, + NULL, 0}, + {4, 16, 2, "\x4a\x65\x66\x65", + "\x77\x68\x61\x74\x20\x64\x6F\x20\x79\x61\x20\x77\x61\x6E\x74\x20\x66\x6F" + "\x72\x20\x6E\x6F\x74\x68\x69\x6E\x67\x3F", + "\x75\x0c\x78\x3e\x6a\xb0\xb5\x03\xea\xa8\x6e\x31\x0a\x5d\xb7\x38", 1, 224, + NULL, 0}, + {16, 16, 3, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", + "\x56\xbe\x34\x52\x1d\x14\x4c\x88\xdb\xb8\xc7\x33\xf0\xe8\xb3\xf6", 1, 400, + NULL, 0}, + {25, 16, 4, + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12" + "\x13\x14\x15\x16\x17\x18\x19", + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", + "\x69\x7e\xaf\x0a\xca\x3a\x3a\xea\x3a\x75\x16\x47\x46\xff\xaa\x79", 1, 400, + NULL, 0}, + {16, 16, 5, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c", + "\x54\x65\x73\x74\x20\x57\x69\x74\x68\x20\x54\x72\x75\x6e\x63\x61\x74\x69" + "\x6f\x6e", + "\x56\x46\x1e\xf2\x34\x2e\xdc\x00\xf9\xba\xb9\x95\x69\x0e\xfd\x4c", 1, 160, + NULL, 0}, + {80, 16, 6, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6E\x67\x20\x4C\x61\x72\x67\x65\x72\x20" + "\x54\x68\x61\x6E\x20\x42\x6C\x6F\x63\x6B\x2D\x53\x69\x7A\x65\x20\x4B\x65" + "\x79\x20\x2D\x20\x48\x61\x73\x68\x20\x4B\x65\x79\x20\x46\x69\x72\x73\x74", + "\x6b\x1a\xb7\xfe\x4b\xd7\xbf\x8f\x0b\x62\xe6\xce\x61\xb9\xd0\xcd", 1, 432, + NULL, 0}, + {80, 16, 7, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6E\x67\x20\x4C\x61\x72\x67\x65\x72\x20" + "\x54\x68\x61\x6E\x20\x42\x6C\x6F\x63\x6B\x2D\x53\x69\x7A\x65\x20\x4B\x65" + "\x79\x20\x61\x6E\x64\x20\x4C\x61\x72\x67\x65\x72\x20\x54\x68\x61\x6E\x20" + "\x4F\x6E\x65\x20\x42\x6C\x6F\x63\x6B\x2D\x53\x69\x7A\x65\x20\x44\x61\x74" + "\x61", + "\x6f\x63\x0f\xad\x67\xcd\xa0\xee\x1f\xb1\xf5\x62\xdb\x3a\xa5\x3e", 1, 584, + NULL, 0}, + {16, 12, 8, + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", + "\x48\x69\x20\x54\x68\x65\x72\x65", + "\x92\x94\x72\x7a\x36\x38\xbb\x1c\x13\xf4\x8e\xf8", 1, 64, NULL, 0}, + {4, 12, 9, "\x4a\x65\x66\x65", + "\x77\x68\x61\x74\x20\x64\x6F\x20\x79\x61\x20\x77\x61\x6E\x74\x20\x66\x6F" + "\x72\x20\x6E\x6F\x74\x68\x69\x6E\x67\x3F", + "\x75\x0c\x78\x3e\x6a\xb0\xb5\x03\xea\xa8\x6e\x31", 1, 224, NULL, 0}, + {16, 12, 10, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", + "\x56\xbe\x34\x52\x1d\x14\x4c\x88\xdb\xb8\xc7\x33", 1, 400, NULL, 0}, + {25, 12, 11, + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12" + "\x13\x14\x15\x16\x17\x18\x19", + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", + "\x69\x7e\xaf\x0a\xca\x3a\x3a\xea\x3a\x75\x16\x47", 1, 400, NULL, 0}, + {16, 12, 12, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c", + "\x54\x65\x73\x74\x20\x57\x69\x74\x68\x20\x54\x72\x75\x6e\x63\x61\x74\x69" + "\x6f\x6e", + "\x56\x46\x1e\xf2\x34\x2e\xdc\x00\xf9\xba\xb9\x95", 1, 160, NULL, 0}, + {80, 12, 13, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6E\x67\x20\x4C\x61\x72\x67\x65\x72\x20" + "\x54\x68\x61\x6E\x20\x42\x6C\x6F\x63\x6B\x2D\x53\x69\x7A\x65\x20\x4B\x65" + "\x79\x20\x2D\x20\x48\x61\x73\x68\x20\x4B\x65\x79\x20\x46\x69\x72\x73\x74", + "\x6b\x1a\xb7\xfe\x4b\xd7\xbf\x8f\x0b\x62\xe6\xce", 1, 432, NULL, 0}, + {80, 12, 14, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6E\x67\x20\x4C\x61\x72\x67\x65\x72\x20" + "\x54\x68\x61\x6E\x20\x42\x6C\x6F\x63\x6B\x2D\x53\x69\x7A\x65\x20\x4B\x65" + "\x79\x20\x61\x6E\x64\x20\x4C\x61\x72\x67\x65\x72\x20\x54\x68\x61\x6E\x20" + "\x4F\x6E\x65\x20\x42\x6C\x6F\x63\x6B\x2D\x53\x69\x7A\x65\x20\x44\x61\x74" + "\x61", + "\x6f\x63\x0f\xad\x67\xcd\xa0\xee\x1f\xb1\xf5\x62", 1, 584, NULL, 0}, + {0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0} +}; diff --git a/test/kat-app/hmac_md5_test.c b/test/kat-app/hmac_md5_test.c new file mode 100644 index 0000000000000000000000000000000000000000..1c78cf469e3f25aa70e34e0533fadb76ad5c767d --- /dev/null +++ b/test/kat-app/hmac_md5_test.c @@ -0,0 +1,249 @@ +/***************************************************************************** + Copyright (c) 2018-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "gcm_ctr_vectors_test.h" +#include "utils.h" +#include "mac_test.h" + +int hmac_md5_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test hmac_md5_test_json[]; + +static int +hmac_md5_job_ok(const struct mac_test *vec, + const struct IMB_JOB *job, + const uint8_t *auth, + const uint8_t *padding, + const size_t sizeof_padding) +{ + if (job->status != IMB_STATUS_COMPLETED) { + printf("line:%d job error status:%d ", __LINE__, job->status); + return 0; + } + + /* hash checks */ + if (memcmp(padding, &auth[sizeof_padding + vec->tagSize], + sizeof_padding)) { + printf("hash overwrite tail\n"); + hexdump(stderr, "Target", + &auth[sizeof_padding + vec->tagSize], + sizeof_padding); + return 0; + } + + if (memcmp(padding, &auth[0], sizeof_padding)) { + printf("hash overwrite head\n"); + hexdump(stderr, "Target", &auth[0], sizeof_padding); + return 0; + } + + if (memcmp(vec->tag, &auth[sizeof_padding], + vec->tagSize)) { + printf("hash mismatched\n"); + hexdump(stderr, "Received", &auth[sizeof_padding], + vec->tagSize); + hexdump(stderr, "Expected", vec->tag, + vec->tagSize); + return 0; + } + return 1; +} + +static int +test_hmac_md5(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const int num_jobs) +{ + struct IMB_JOB *job; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + int i = 0, jobs_rx = 0, ret = -1; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_MD5_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_MD5_DIGEST_SIZE_IN_BYTES], 16); + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_MD5, vec->key, vec->keySize, + ipad_hash, opad_hash); + + /* empty the manager */ + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = (const void *) vec->msg; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_MD5; + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job) { + jobs_rx++; + /* + * HMAC-MD5 requires 8 submissions to get one back + */ + if (num_jobs < 8) { + printf("%d Unexpected return from submit_job\n", + __LINE__); + goto end; + } + if (!hmac_md5_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + jobs_rx++; + if (!hmac_md5_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + /* empty the manager before next tests */ + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static void +test_hmac_md5_std_vectors(struct IMB_MGR *mb_mgr, + const int num_jobs, + struct test_suite_context *ts) +{ + + const struct mac_test *v = hmac_md5_test_json; + + if (!quiet_mode) + printf("HMAC-MD5 standard test vectors (N jobs = %d):\n", + num_jobs); + for (; v->msg != NULL; v++) { + if (!quiet_mode) { +#ifdef DEBUG + printf("RFC2202 Test Case %zu key_len:%zu " + "data_len:%zu digest_len:%zu\n", + v->tcId, + v->keySize, + v->msgSize / 8, + v->tagSize); +#else + printf("."); +#endif + } + /* No functionality for keys larger than block size */ + if (v->keySize > IMB_MD5_BLOCK_SIZE) { +#ifdef DEBUG + if (!quiet_mode) + printf("Skipped vector %zu, " + "Key size larger than block size\n", + v->tcId); +#endif + continue; + } + if (test_hmac_md5(mb_mgr, v, num_jobs)) { + printf("error #%zu\n", v->tcId); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + } + if (!quiet_mode) + printf("\n"); +} + +int +hmac_md5_test(struct IMB_MGR *mb_mgr) +{ + struct test_suite_context ts; + int num_jobs, errors = 0; + + test_suite_start(&ts, "HMAC-MD5"); + for (num_jobs = 1; num_jobs <= 17; num_jobs++) + test_hmac_md5_std_vectors(mb_mgr, num_jobs, &ts); + errors = test_suite_end(&ts); + + return errors; +} diff --git a/test/kat-app/hmac_sha1.json.c b/test/kat-app/hmac_sha1.json.c new file mode 100644 index 0000000000000000000000000000000000000000..2d7573559d71996d641e8dc1b9cbc3784fc2d4c6 --- /dev/null +++ b/test/kat-app/hmac_sha1.json.c @@ -0,0 +1,121 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "mac_test.h" + +const struct mac_test hmac_sha1_test_kat_json[] = { + /* + * Test vectors from https://tools.ietf.org/html/rfc2202 + */ + { 20, 20, 1, + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b", + "\x48\x69\x20\x54\x68\x65\x72\x65", + "\xb6\x17\x31\x86\x55\x05\x72\x64\xe2\x8b\xc0\xb6\xfb\x37\x8c\x8e\xf1" + "\x46\xbe\x00", + 1, 64, NULL, 0 }, + { 4, 20, 2, "\x4a\x65\x66\x65", + "\x77\x68\x61\x74\x20\x64\x6F\x20\x79\x61\x20\x77\x61\x6E\x74\x20\x66" + "\x6F\x72\x20\x6E\x6F\x74\x68\x69\x6E\x67\x3F", + "\xef\xfc\xdf\x6a\xe5\xeb\x2f\xa2\xd2\x74\x16\xd5\xf1\x84\xdf\x9c\x25" + "\x9a\x7c\x79", + 1, 224, NULL, 0 }, + { 20, 20, 3, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", + "\x12\x5d\x73\x42\xb9\xac\x11\xcd\x91\xa3\x9a\xf4\x8a\xa1\x7b\x4f\x63" + "\xf1\x75\xd3", + 1, 400, NULL, 0 }, + { 25, 20, 4, + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19", + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", + "\x4c\x90\x07\xf4\x02\x62\x50\xc6\xbc\x84\x14\xf9\xbf\x50\xc8\x6c\x2d" + "\x72\x35\xda", + 1, 400, NULL, 0 }, + { 20, 20, 5, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c" + "\x0c\x0c\x0c", + "\x54\x65\x73\x74\x20\x57\x69\x74\x68\x20\x54\x72\x75\x6e\x63\x61\x74" + "\x69\x6f\x6e", + "\x4c\x1a\x03\x42\x4b\x55\xe0\x7f\xe7\xf2\x7b\xe1\xd5\x8b\xb9\x32\x4a" + "\x9a\x5a\x04", + 1, 160, NULL, 0 }, + { 20, 12, 6, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c" + "\x0c\x0c\x0c", + "\x54\x65\x73\x74\x20\x57\x69\x74\x68\x20\x54\x72\x75\x6e\x63\x61\x74" + "\x69\x6f\x6e", + "\x4c\x1a\x03\x42\x4b\x55\xe0\x7f\xe7\xf2\x7b\xe1", + 1, 160, NULL, 0 }, + { 80, 20, 7, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6e\x67\x20\x4c\x61\x72\x67\x65\x72" + "\x20\x54\x68\x61\x6e\x20\x42\x6c\x6f\x63\x6b\x2d\x53\x69\x7a\x65\x20" + "\x4b\x65\x79\x20\x2d\x20\x48\x61\x73\x68\x20\x4b\x65\x79\x20\x46\x69" + "\x72\x73\x74", + "\xaa\x4a\xe5\xe1\x52\x72\xd0\x0e\x95\x70\x56\x37\xce\x8a\x3b\x55\xed" + "\x40\x21\x12", + 1, 432, NULL, 0 }, + { 80, 20, 8, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6e\x67\x20\x4c\x61\x72\x67\x65\x72" + "\x20\x54\x68\x61\x6e\x20\x42\x6c\x6f\x63\x6b\x2d\x53\x69\x7a\x65\x20" + "\x4b\x65\x79\x20\x61\x6e\x64\x20\x4c\x61\x72\x67\x65\x72\x20\x54\x68" + "\x61\x6e\x20\x4f\x6e\x65\x20\x42\x6c\x6f\x63\x6b\x2d\x53\x69\x7a\x65" + "\x20\x44\x61\x74\x61", + "\xe8\xe9\x9d\x0f\x45\x23\x7d\x78\x6d\x6b\xba\xa7\x96\x5c\x78\x08\xbb" + "\xff\x1a\x91", + 1, 584, NULL, 0 }, + /* + * Test vector from + * https://csrc.nist.gov/csrc/media/publications/fips/198/ + * archive/2002-03-06/documents/fips-198a.pdf + */ + { 49, 12, 9, + "\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80" + "\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91" + "\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0", + "\x53\x61\x6d\x70\x6c\x65\x20\x23\x34", + "\x9e\xa8\x86\xef\xe2\x68\xdb\xec\xce\x42\x0c\x75", 1, 72, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/kat-app/hmac_sha1_test.c b/test/kat-app/hmac_sha1_test.c new file mode 100644 index 0000000000000000000000000000000000000000..63c38a9e01380e4fef9491389e3113a72e712696 --- /dev/null +++ b/test/kat-app/hmac_sha1_test.c @@ -0,0 +1,484 @@ +/***************************************************************************** + Copyright (c) 2018-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "gcm_ctr_vectors_test.h" +#include "utils.h" +#include "mac_test.h" + +int hmac_sha1_test(struct IMB_MGR *mb_mgr); + +#define DIGEST96_SIZE 12 +#define MAX_BURST_JOBS 32 + +extern const struct mac_test hmac_sha1_test_kat_json[]; +static int +hmac_sha1_job_ok(const struct mac_test *vec, + const struct IMB_JOB *job, + const uint8_t *auth, + const uint8_t *padding, + const size_t sizeof_padding) +{ + if (job->status != IMB_STATUS_COMPLETED) { + printf("line:%d job error status:%d ", __LINE__, job->status); + return 0; + } + + /* hash checks */ + if (memcmp(padding, &auth[sizeof_padding + vec->tagSize], + sizeof_padding)) { + printf("hash overwrite tail\n"); + hexdump(stderr, "Target", + &auth[sizeof_padding + vec->tagSize], + sizeof_padding); + return 0; + } + + if (memcmp(padding, &auth[0], sizeof_padding)) { + printf("hash overwrite head\n"); + hexdump(stderr, "Target", &auth[0], sizeof_padding); + return 0; + } + + if (memcmp(vec->tag, &auth[sizeof_padding], + vec->tagSize)) { + printf("hash mismatched\n"); + hexdump(stderr, "Received", &auth[sizeof_padding], + vec->tagSize); + hexdump(stderr, "Expected", vec->tag, + vec->tagSize); + return 0; + } + return 1; +} + +static int +test_hmac_sha1(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const uint32_t num_jobs) +{ + struct IMB_JOB *job; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0; + int ret = -1; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_1, + vec->key, vec->keySize, ipad_hash, opad_hash); + + for (i = 0; i < num_jobs; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = (const void *) vec->msg; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_1; + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job) { + jobs_rx++; + /* + * SHANI HMAC-SHA implementation can return a completed + * job after 2nd submission + */ + if (num_jobs < 2) { + printf("%d Unexpected return from submit_job\n", + __LINE__); + goto end; + } + if (!hmac_sha1_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + jobs_rx++; + if (!hmac_sha1_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + + if (jobs_rx != num_jobs) { + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + /* empty the manager before next tests */ + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static int +test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const uint32_t num_jobs) +{ + struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = {NULL}; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0; + int ret = -1, err; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + uint32_t completed_jobs = 0; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_1, + vec->key, vec->keySize, ipad_hash, opad_hash); + + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = (const void *) vec->msg; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_1; + + job->user_data = auths[i]; + + imb_set_session(mb_mgr, job); + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + +check_burst_jobs: + for (i = 0; i < completed_jobs; i++) { + job = jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i+1); + goto end; + } + + if (!hmac_sha1_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, + num_jobs - completed_jobs, + jobs); + if (completed_jobs == 0) { + printf("Expected %u jobs, received %u\n", + num_jobs, jobs_rx); + goto end; + } + goto check_burst_jobs; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static int +test_hmac_sha1_hash_burst(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const uint32_t num_jobs) +{ + struct IMB_JOB *job, jobs[MAX_BURST_JOBS] = {0}; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0; + int ret = -1; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + uint32_t completed_jobs = 0; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + imb_hmac_ipad_opad(mb_mgr, IMB_AUTH_HMAC_SHA_1, + vec->key, vec->keySize, ipad_hash, opad_hash); + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = (const void *) vec->msg; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_1; + + job->user_data = auths[i]; + + } + + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, + IMB_AUTH_HMAC_SHA_1); + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i+1); + goto end; + } + + if (!hmac_sha1_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static void +test_hmac_sha1_std_vectors(struct IMB_MGR *mb_mgr, + const uint32_t num_jobs, + struct test_suite_context *ts) +{ + const struct mac_test *v = hmac_sha1_test_kat_json; + + if (!quiet_mode) + printf("HMAC-SHA1 standard test vectors (N jobs = %u):\n", num_jobs); + while (v->msg != NULL) { + if (!quiet_mode) { +#ifdef DEBUG + printf("RFC2202 Test Case %zu keySize:%zu " + "msgSize:%zu tagSize:%zu\n", + v->tcId, + v->keySize, + v->msgSize / 8, + v->tagSize); +#else + printf("."); +#endif + } + + if (test_hmac_sha1(mb_mgr, v, num_jobs)) { + printf("error #%zu\n", v->tcId); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + if (test_hmac_sha1_burst(mb_mgr, v, + num_jobs)) { + printf("error #%zu - burst API\n", v->tcId); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + if (test_hmac_sha1_hash_burst(mb_mgr, v, + num_jobs)) { + printf("error #%zu - hash-only burst API\n", v->tcId); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + + v++; + } + if (!quiet_mode) + printf("\n"); +} + +int +hmac_sha1_test(struct IMB_MGR *mb_mgr) +{ + struct test_suite_context ts; + int errors = 0; + uint32_t num_jobs; + + test_suite_start(&ts, "HMAC-SHA1"); + for (num_jobs = 1; num_jobs <= MAX_BURST_JOBS; num_jobs++) + test_hmac_sha1_std_vectors(mb_mgr, num_jobs, &ts); + errors = test_suite_end(&ts); + + return errors; +} diff --git a/test/kat-app/hmac_sha224.json.c b/test/kat-app/hmac_sha224.json.c new file mode 100644 index 0000000000000000000000000000000000000000..a928651a15f79e4314411f2a7b40e5ea5886b8c4 --- /dev/null +++ b/test/kat-app/hmac_sha224.json.c @@ -0,0 +1,120 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* HMACSHA224, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha224_test_kat_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc4231 */ + { 20, 28, 1, + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b", + "\x48\x69\x20\x54\x68\x65\x72\x65", + "\x89\x6f\xb1\x12\x8a\xbb\xdf\x19\x68\x32\x10\x7c\xd4\x9d\xf3\x3f\x47" + "\xb4\xb1\x16\x99\x12\xba\x4f\x53\x68\x4b\x22", + 1, 64, NULL, 0 }, + { 4, 28, 2, "\x4a\x65\x66\x65", + "\x77\x68\x61\x74\x20\x64\x6f\x20\x79\x61\x20\x77\x61\x6e\x74\x20\x66" + "\x6f\x72\x20\x6e\x6f\x74\x68\x69\x6e\x67\x3f", + "\xa3\x0e\x01\x09\x8b\xc6\xdb\xbf\x45\x69\x0f\x3a\x7e\x9e\x6d\x0f\x8b" + "\xbe\xa2\xa3\x9e\x61\x48\x00\x8f\xd0\x5e\x44", + 1, 224, NULL, 0 }, + { 20, 28, 3, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", + "\x7f\xb3\xcb\x35\x88\xc6\xc1\xf6\xff\xa9\x69\x4d\x7d\x6a\xd2\x64\x93" + "\x65\xb0\xc1\xf6\x5d\x69\xd1\xec\x83\x33\xea", + 1, 400, NULL, 0 }, + { 25, 28, 4, + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19", + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", + "\x6c\x11\x50\x68\x74\x01\x3c\xac\x6a\x2a\xbc\x1b\xb3\x82\x62\x7c\xec" + "\x6a\x90\xd8\x6e\xfc\x01\x2d\xe7\xaf\xec\x5a", + 1, 400, NULL, 0 }, + { 20, 16, 5, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c" + "\x0c\x0c\x0c", + "\x54\x65\x73\x74\x20\x57\x69\x74\x68\x20\x54\x72\x75\x6e\x63\x61\x74" + "\x69\x6f\x6e", + "\x0e\x2a\xea\x68\xa9\x0c\x8d\x37\xc9\x88\xbc\xdb\x9f\xca\x6f\xa8", + 1, 160, NULL, 0 }, + { 131, 28, 6, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6e\x67\x20\x4c\x61\x72\x67\x65\x72" + "\x20\x54\x68\x61\x6e\x20\x42\x6c\x6f\x63\x6b\x2d\x53\x69\x7a\x65\x20" + "\x4b\x65\x79\x20\x2d\x20\x48\x61\x73\x68\x20\x4b\x65\x79\x20\x46\x69" + "\x72\x73\x74", + "\x95\xe9\xa0\xdb\x96\x20\x95\xad\xae\xbe\x9b\x2d\x6f\x0d\xbc\xe2\xd4" + "\x99\xf1\x12\xf2\xd2\xb7\x27\x3f\xa6\x87\x0e", + 1, 432, NULL, 0 }, + { 131, 28, 7, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x74\x65\x73\x74\x20\x75\x73" + "\x69\x6e\x67\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e" + "\x20\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x6b\x65\x79\x20\x61" + "\x6e\x64\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e\x20" + "\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x64\x61\x74\x61\x2e\x20" + "\x54\x68\x65\x20\x6b\x65\x79\x20\x6e\x65\x65\x64\x73\x20\x74\x6f\x20" + "\x62\x65\x20\x68\x61\x73\x68\x65\x64\x20\x62\x65\x66\x6f\x72\x65\x20" + "\x62\x65\x69\x6e\x67\x20\x75\x73\x65\x64\x20\x62\x79\x20\x74\x68\x65" + "\x20\x48\x4d\x41\x43\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x2e", + "\x3a\x85\x41\x66\xac\x5d\x9f\x02\x3f\x54\xd5\x17\xd0\xb3\x9d\xbd\x94" + "\x67\x70\xdb\x9c\x2b\x95\xc9\xf6\xf5\x65\xd1", + 1, 1216, NULL, 0 }, + /* Vectors from https://csrc.nist.gov/csrc/media/projects/ */ + { 64, 28, 8, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x53\x61\x6d\x70\x6c\x65\x20\x6d\x65\x73\x73\x61\x67\x65\x20\x66\x6f" + "\x72\x20\x6b\x65\x79\x6c\x65\x6e\x3d\x62\x6c\x6f\x63\x6b\x6c\x65" + "\x6e", + "\xc7\x40\x5e\x3a\xe0\x58\xe8\xcd\x30\xb0\x8b\x41\x40\x24\x85\x81\xed" + "\x17\x4c\xb3\x4e\x12\x24\xbc\xc1\xef\xc8\x1b", + 1, 272, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/kat-app/hmac_sha256.json.c b/test/kat-app/hmac_sha256.json.c new file mode 100644 index 0000000000000000000000000000000000000000..1fa1a90a76731b460aa60338d80d4cdeea9a4450 --- /dev/null +++ b/test/kat-app/hmac_sha256.json.c @@ -0,0 +1,120 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* HMACSHA256, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha256_test_kat_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc4231 */ + { 20, 32, 1, + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b", + "\x48\x69\x20\x54\x68\x65\x72\x65", + "\xb0\x34\x4c\x61\xd8\xdb\x38\x53\x5c\xa8\xaf\xce\xaf\x0b\xf1\x2b\x88" + "\x1d\xc2\x00\xc9\x83\x3d\xa7\x26\xe9\x37\x6c\x2e\x32\xcf\xf7", + 1, 64, NULL, 0 }, + { 4, 32, 2, "\x4a\x65\x66\x65", + "\x77\x68\x61\x74\x20\x64\x6f\x20\x79\x61\x20\x77\x61\x6e\x74\x20\x66" + "\x6f\x72\x20\x6e\x6f\x74\x68\x69\x6e\x67\x3f", + "\x5b\xdc\xc1\x46\xbf\x60\x75\x4e\x6a\x04\x24\x26\x08\x95\x75\xc7\x5a" + "\x00\x3f\x08\x9d\x27\x39\x83\x9d\xec\x58\xb9\x64\xec\x38\x43", + 1, 224, NULL, 0 }, + { 20, 32, 3, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", + "\x77\x3e\xa9\x1e\x36\x80\x0e\x46\x85\x4d\xb8\xeb\xd0\x91\x81\xa7\x29" + "\x59\x09\x8b\x3e\xf8\xc1\x22\xd9\x63\x55\x14\xce\xd5\x65\xfe", + 1, 400, NULL, 0 }, + { 25, 32, 4, + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19", + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", + "\x82\x55\x8a\x38\x9a\x44\x3c\x0e\xa4\xcc\x81\x98\x99\xf2\x08\x3a\x85" + "\xf0\xfa\xa3\xe5\x78\xf8\x07\x7a\x2e\x3f\xf4\x67\x29\x66\x5b", + 1, 400, NULL, 0 }, + { 20, 16, 5, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c" + "\x0c\x0c\x0c", + "\x54\x65\x73\x74\x20\x57\x69\x74\x68\x20\x54\x72\x75\x6e\x63\x61\x74" + "\x69\x6f\x6e", + "\xa3\xb6\x16\x74\x73\x10\x0e\xe0\x6e\x0c\x79\x6c\x29\x55\x55\x2b", + 1, 160, NULL, 0 }, + { 131, 32, 6, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6e\x67\x20\x4c\x61\x72\x67\x65\x72" + "\x20\x54\x68\x61\x6e\x20\x42\x6c\x6f\x63\x6b\x2d\x53\x69\x7a\x65\x20" + "\x4b\x65\x79\x20\x2d\x20\x48\x61\x73\x68\x20\x4b\x65\x79\x20\x46\x69" + "\x72\x73\x74", + "\x60\xe4\x31\x59\x1e\xe0\xb6\x7f\x0d\x8a\x26\xaa\xcb\xf5\xb7\x7f\x8e" + "\x0b\xc6\x21\x37\x28\xc5\x14\x05\x46\x04\x0f\x0e\xe3\x7f\x54", + 1, 432, NULL, 0 }, + { 131, 32, 7, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x74\x65\x73\x74\x20\x75\x73" + "\x69\x6e\x67\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e" + "\x20\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x6b\x65\x79\x20\x61" + "\x6e\x64\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e\x20" + "\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x64\x61\x74\x61\x2e\x20" + "\x54\x68\x65\x20\x6b\x65\x79\x20\x6e\x65\x65\x64\x73\x20\x74\x6f\x20" + "\x62\x65\x20\x68\x61\x73\x68\x65\x64\x20\x62\x65\x66\x6f\x72\x65\x20" + "\x62\x65\x69\x6e\x67\x20\x75\x73\x65\x64\x20\x62\x79\x20\x74\x68\x65" + "\x20\x48\x4d\x41\x43\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x2e", + "\x9b\x09\xff\xa7\x1b\x94\x2f\xcb\x27\x63\x5f\xbc\xd5\xb0\xe9\x44\xbf" + "\xdc\x63\x64\x4f\x07\x13\x93\x8a\x7f\x51\x53\x5c\x3a\x35\xe2", + 1, 1216, NULL, 0 }, + /* Vectors from https://csrc.nist.gov/csrc/media/projects/ */ + { 64, 32, 8, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x53\x61\x6d\x70\x6c\x65\x20\x6d\x65\x73\x73\x61\x67\x65\x20\x66\x6f" + "\x72\x20\x6b\x65\x79\x6c\x65\x6e\x3d\x62\x6c\x6f\x63\x6b\x6c\x65" + "\x6e", + "\x8b\xb9\xa1\xdb\x98\x06\xf2\x0d\xf7\xf7\x7b\x82\x13\x8c\x79\x14\xd1" + "\x74\xd5\x9e\x13\xdc\x4d\x01\x69\xc9\x05\x7b\x13\x3e\x1d\x62", + 1, 272, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/kat-app/hmac_sha256_sha512_test.c b/test/kat-app/hmac_sha256_sha512_test.c new file mode 100644 index 0000000000000000000000000000000000000000..44dbbbb7851e1dbfee639a6ea1f547d968cd4883 --- /dev/null +++ b/test/kat-app/hmac_sha256_sha512_test.c @@ -0,0 +1,676 @@ +/***************************************************************************** + Copyright (c) 2018-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "gcm_ctr_vectors_test.h" +#include "utils.h" +#include "mac_test.h" + +#define max_burst_jobs 32 + +int hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test hmac_sha224_test_kat_json[]; +extern const struct mac_test hmac_sha256_test_kat_json[]; +extern const struct mac_test hmac_sha384_test_kat_json[]; +extern const struct mac_test hmac_sha512_test_kat_json[]; + +static int +hmac_shax_job_ok(const struct mac_test *vec, + const struct IMB_JOB *job, + const int sha_type, + const uint8_t *auth, + const uint8_t *padding, + const size_t sizeof_padding) +{ + const uint8_t *p_digest = NULL; + + switch (sha_type) { + case 224: + case 256: + case 384: + case 512: + p_digest = (const void *) vec->tag; + break; + default: + printf("line:%d wrong SHA type 'SHA-%d' ", __LINE__, sha_type); + return 0; + break; + } + + if (job->status != IMB_STATUS_COMPLETED) { + printf("line:%d job error status:%d ", __LINE__, job->status); + return 0; + } + + /* hash checks */ + if (memcmp(padding, &auth[sizeof_padding + vec->tagSize], + sizeof_padding)) { + printf("hash overwrite tail\n"); + hexdump(stderr, "Target", + &auth[sizeof_padding + vec->tagSize], + sizeof_padding); + return 0; + } + + if (memcmp(padding, &auth[0], sizeof_padding)) { + printf("hash overwrite head\n"); + hexdump(stderr, "Target", &auth[0], sizeof_padding); + return 0; + } + + if (memcmp(p_digest, &auth[sizeof_padding], vec->tagSize)) { + printf("hash mismatched\n"); + hexdump(stderr, "Received", &auth[sizeof_padding], vec->tagSize); + hexdump(stderr, "Expected", p_digest, vec->tagSize); + return 0; + } + return 1; +} + +static int +test_hmac_shax(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const uint32_t num_jobs, + const int sha_type) +{ + struct IMB_JOB *job; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0; + int ret = -1; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + IMB_HASH_ALG hash_type; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + switch (sha_type) { + case 224: + case 256: + case 384: + case 512: + break; + default: + fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", + sha_type); + goto end2; + } + + switch (sha_type) { + case 224: + hash_type = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + hash_type = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + hash_type = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + hash_type = IMB_AUTH_HMAC_SHA_512; + break; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + imb_hmac_ipad_opad(mb_mgr, hash_type, vec->key, + vec->keySize, ipad_hash, opad_hash); + + /* empty the manager */ + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = (const void *) vec->msg; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + + switch (sha_type) { + case 224: + job->hash_alg = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + job->hash_alg = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + job->hash_alg = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + job->hash_alg = IMB_AUTH_HMAC_SHA_512; + break; + } + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job) { + jobs_rx++; + /* + * SHANI HMAC-SHA implementation can return a completed + * job after 2nd submission + */ + if (num_jobs < 2) { + printf("%d Unexpected return from submit_job\n", + __LINE__); + goto end; + } + if (!hmac_shax_job_ok(vec, job, sha_type, + job->user_data, + padding, sizeof(padding))) + goto end; + } + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + jobs_rx++; + if (!hmac_shax_job_ok(vec, job, sha_type, + job->user_data, + padding, sizeof(padding))) + goto end; + } + + if (jobs_rx != num_jobs) { + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static int +test_hmac_shax_burst(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const uint32_t num_jobs, + const int sha_type) +{ + struct IMB_JOB *job, *jobs[max_burst_jobs] = {NULL}; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0, completed_jobs = 0; + int ret = -1, err; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + IMB_HASH_ALG hash_type; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + switch (sha_type) { + case 224: + case 256: + case 384: + case 512: + break; + default: + fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", + sha_type); + goto end2; + } + + switch (sha_type) { + case 224: + hash_type = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + hash_type = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + hash_type = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + hash_type = IMB_AUTH_HMAC_SHA_512; + break; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + imb_hmac_ipad_opad(mb_mgr, hash_type, vec->key, + vec->keySize, ipad_hash, opad_hash); + + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = (const void *) vec->msg; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + + switch (sha_type) { + case 224: + job->hash_alg = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + job->hash_alg = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + job->hash_alg = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + job->hash_alg = IMB_AUTH_HMAC_SHA_512; + break; + } + + job->user_data = auths[i]; + + imb_set_session(mb_mgr, job); + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + + check_burst_jobs: + for (i = 0; i < completed_jobs; i++) { + job = jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i+1); + goto end; + } + + if (!hmac_shax_job_ok(vec, job, sha_type, + job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, + num_jobs - completed_jobs, + jobs); + if (completed_jobs == 0) { + printf("Expected %u jobs, received %u\n", + num_jobs, jobs_rx); + goto end; + } + goto check_burst_jobs; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static int +test_hmac_shax_hash_burst(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const uint32_t num_jobs, + const int sha_type) +{ + struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; + uint8_t padding[16]; + uint8_t **auths = NULL; + uint32_t i = 0, jobs_rx = 0, completed_jobs = 0; + int ret = -1; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + IMB_HASH_ALG hash_type; + + if (num_jobs == 0) + return 0; + + auths = malloc(num_jobs * sizeof(void *)); + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + switch (sha_type) { + case 224: + case 256: + case 384: + case 512: + break; + default: + fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", + sha_type); + goto end2; + } + + switch (sha_type) { + case 224: + hash_type = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + hash_type = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + hash_type = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + hash_type = IMB_AUTH_HMAC_SHA_512; + break; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + imb_hmac_ipad_opad(mb_mgr, hash_type, vec->key, + vec->keySize, ipad_hash, opad_hash); + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = (const void *) vec->msg; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + + switch (sha_type) { + case 224: + job->hash_alg = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + job->hash_alg = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + job->hash_alg = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + job->hash_alg = IMB_AUTH_HMAC_SHA_512; + break; + } + + job->user_data = auths[i]; + + } + + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, + job->hash_alg); + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i+1); + goto end; + } + + if (!hmac_shax_job_ok(vec, job, sha_type, + job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static void +test_hmac_shax_std_vectors(struct IMB_MGR *mb_mgr, + const int sha_type, + const uint32_t num_jobs, + struct test_suite_context *ts) +{ + const struct mac_test *v; + + switch (sha_type) { + case 224: + v = hmac_sha224_test_kat_json; + break; + case 256: + v = hmac_sha256_test_kat_json; + break; + case 384: + v = hmac_sha384_test_kat_json; + break; + default: + v = hmac_sha512_test_kat_json; + break; + } + if (!quiet_mode) + printf("HMAC-SHA%d standard test vectors (N jobs = %u):\n", + sha_type, num_jobs); + for (; v->msg != NULL; v++) { + if (!quiet_mode) { +#ifdef DEBUG + printf("RFC4231 Test Case %zu key_len:%zu " + "data_len:%zu\n", + v->tcId, + v->keySize, + v->msgSize / 8); +#else + printf("."); +#endif + } + /* @todo add truncation functionality to hmac_sha224 to hmac_sha 512*/ + const int flag = + ((sha_type == 224 && v->tagSize != IMB_SHA224_DIGEST_SIZE_IN_BYTES) || + (sha_type == 256 && v->tagSize != IMB_SHA256_DIGEST_SIZE_IN_BYTES) || + (sha_type == 384 && v->tagSize != IMB_SHA384_DIGEST_SIZE_IN_BYTES) || + (sha_type == 512 && v->tagSize != IMB_SHA512_DIGEST_SIZE_IN_BYTES)); + + if (flag) { +#ifdef DEBUG + if (!quiet_mode) + printf("Skipped vector %zu, " + "N/A for HMAC-SHA%d\n", + v->tcId, sha_type); +#endif + continue; + } + if (test_hmac_shax(mb_mgr, v, num_jobs, sha_type)) { + printf("error #%zu\n", v->tcId); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + if (test_hmac_shax_burst(mb_mgr, v, num_jobs, sha_type)) { + printf("error #%zu - burst API\n", v->tcId); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + if (test_hmac_shax_hash_burst(mb_mgr, v, num_jobs, sha_type)) { + printf("error #%zu - hash-only burst API\n", v->tcId); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + } + if (!quiet_mode) + printf("\n"); +} + +int +hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr) +{ + const int sha_types_tab[] = { + 224, 256, 384, 512 + }; + static const char * const sha_names_tab[] = { + "HMAC-SHA224", "HMAC-SHA256", "HMAC-SHA384", "HMAC-SHA512" + }; + unsigned i, num_jobs; + int errors = 0; + + for (i = 0; i < DIM(sha_types_tab); i++) { + struct test_suite_context ts; + + test_suite_start(&ts, sha_names_tab[i]); + for (num_jobs = 1; num_jobs <= max_burst_jobs; num_jobs++) + test_hmac_shax_std_vectors(mb_mgr, sha_types_tab[i], + num_jobs, &ts); + errors += test_suite_end(&ts); + } + + return errors; +} diff --git a/test/kat-app/hmac_sha384.json.c b/test/kat-app/hmac_sha384.json.c new file mode 100644 index 0000000000000000000000000000000000000000..eb95f2868b3cf34925123e2968394db4b08c7bee --- /dev/null +++ b/test/kat-app/hmac_sha384.json.c @@ -0,0 +1,125 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* HMACSHA384, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha384_test_kat_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc4231 */ + { 20, 48, 1, + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b", + "\x48\x69\x20\x54\x68\x65\x72\x65", + "\xaf\xd0\x39\x44\xd8\x48\x95\x62\x6b\x08\x25\xf4\xab\x46\x90\x7f\x15" + "\xf9\xda\xdb\xe4\x10\x1e\xc6\x82\xaa\x03\x4c\x7c\xeb\xc5\x9c\xfa\xea" + "\x9e\xa9\x07\x6e\xde\x7f\x4a\xf1\x52\xe8\xb2\xfa\x9c\xb6", + 1, 64, NULL, 0 }, + { 4, 48, 2, "\x4a\x65\x66\x65", + "\x77\x68\x61\x74\x20\x64\x6f\x20\x79\x61\x20\x77\x61\x6e\x74\x20\x66" + "\x6f\x72\x20\x6e\x6f\x74\x68\x69\x6e\x67\x3f", + "\xaf\x45\xd2\xe3\x76\x48\x40\x31\x61\x7f\x78\xd2\xb5\x8a\x6b\x1b\x9c" + "\x7e\xf4\x64\xf5\xa0\x1b\x47\xe4\x2e\xc3\x73\x63\x22\x44\x5e\x8e\x22" + "\x40\xca\x5e\x69\xe2\xc7\x8b\x32\x39\xec\xfa\xb2\x16\x49", + 1, 224, NULL, 0 }, + { 20, 48, 3, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", + "\x88\x06\x26\x08\xd3\xe6\xad\x8a\x0a\xa2\xac\xe0\x14\xc8\xa8\x6f\x0a" + "\xa6\x35\xd9\x47\xac\x9f\xeb\xe8\x3e\xf4\xe5\x59\x66\x14\x4b\x2a\x5a" + "\xb3\x9d\xc1\x38\x14\xb9\x4e\x3a\xb6\xe1\x01\xa3\x4f\x27", + 1, 400, NULL, 0 }, + { 25, 48, 4, + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19", + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", + "\x3e\x8a\x69\xb7\x78\x3c\x25\x85\x19\x33\xab\x62\x90\xaf\x6c\xa7\x7a" + "\x99\x81\x48\x08\x50\x00\x9c\xc5\x57\x7c\x6e\x1f\x57\x3b\x4e\x68\x01" + "\xdd\x23\xc4\xa7\xd6\x79\xcc\xf8\xa3\x86\xc6\x74\xcf\xfb", + 1, 400, NULL, 0 }, + { 20, 16, 5, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c" + "\x0c\x0c\x0c", + "\x3a\xbf\x34\xc3\x50\x3b\x2a\x23\xa4\x6e\xfc\x61\x9b\xae\xf8\x97", + "\xa3\xb6\x16\x74\x73\x10\x0e\xe0\x6e\x0c\x79\x6c\x29\x55\x55\x2b", + 1, 128, NULL, 0 }, + { 131, 48, 6, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6e\x67\x20\x4c\x61\x72\x67\x65\x72" + "\x20\x54\x68\x61\x6e\x20\x42\x6c\x6f\x63\x6b\x2d\x53\x69\x7a\x65\x20" + "\x4b\x65\x79\x20\x2d\x20\x48\x61\x73\x68\x20\x4b\x65\x79\x20\x46\x69" + "\x72\x73\x74", + "\x4e\xce\x08\x44\x85\x81\x3e\x90\x88\xd2\xc6\x3a\x04\x1b\xc5\xb4\x4f" + "\x9e\xf1\x01\x2a\x2b\x58\x8f\x3c\xd1\x1f\x05\x03\x3a\xc4\xc6\x0c\x2e" + "\xf6\xab\x40\x30\xfe\x82\x96\x24\x8d\xf1\x63\xf4\x49\x52", + 1, 432, NULL, 0 }, + { 131, 48, 7, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x74\x65\x73\x74\x20\x75\x73" + "\x69\x6e\x67\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e" + "\x20\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x6b\x65\x79\x20\x61" + "\x6e\x64\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e\x20" + "\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x64\x61\x74\x61\x2e\x20" + "\x54\x68\x65\x20\x6b\x65\x79\x20\x6e\x65\x65\x64\x73\x20\x74\x6f\x20" + "\x62\x65\x20\x68\x61\x73\x68\x65\x64\x20\x62\x65\x66\x6f\x72\x65\x20" + "\x62\x65\x69\x6e\x67\x20\x75\x73\x65\x64\x20\x62\x79\x20\x74\x68\x65" + "\x20\x48\x4d\x41\x43\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x2e", + "\x66\x17\x17\x8e\x94\x1f\x02\x0d\x35\x1e\x2f\x25\x4e\x8f\xd3\x2c\x60" + "\x24\x20\xfe\xb0\xb8\xfb\x9a\xdc\xce\xbb\x82\x46\x1e\x99\xc5\xa6\x78" + "\xcc\x31\xe7\x99\x17\x6d\x38\x60\xe6\x11\x0c\x46\x52\x3e", + 1, 1216, NULL, 0 }, + /* Vectors from https://csrc.nist.gov/csrc/media/projects/ */ + { 64, 24, 8, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x53\x61\x6d\x70\x6c\x65\x20\x6d\x65\x73\x73\x61\x67\x65\x20\x66\x6f" + "\x72\x20\x6b\x65\x79\x6c\x65\x6e\x3d\x62\x6c\x6f\x63\x6b\x6c\x65" + "\x6e", + "\x63\xc5\xda\xa5\xe6\x51\x84\x7c\xa8\x97\xc9\x58\x14\xab\x83\x0b\xed" + "\xed\xc7\xd2\x5e\x83\xee\xf9", + 1, 272, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/kat-app/hmac_sha512.json.c b/test/kat-app/hmac_sha512.json.c new file mode 100644 index 0000000000000000000000000000000000000000..e172266582918ccad13300cea8cc33393801c4e9 --- /dev/null +++ b/test/kat-app/hmac_sha512.json.c @@ -0,0 +1,131 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* HMACSHA512, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha512_test_kat_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc4231 */ + { 20, 64, 1, + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b", + "\x48\x69\x20\x54\x68\x65\x72\x65", + "\x87\xaa\x7c\xde\xa5\xef\x61\x9d\x4f\xf0\xb4\x24\x1a\x1d\x6c\xb0\x23" + "\x79\xf4\xe2\xce\x4e\xc2\x78\x7a\xd0\xb3\x05\x45\xe1\x7c\xde\xda\xa8" + "\x33\xb7\xd6\xb8\xa7\x02\x03\x8b\x27\x4e\xae\xa3\xf4\xe4\xbe\x9d\x91" + "\x4e\xeb\x61\xf1\x70\x2e\x69\x6c\x20\x3a\x12\x68\x54", + 1, 64, NULL, 0 }, + { 4, 64, 2, "\x4a\x65\x66\x65", + "\x77\x68\x61\x74\x20\x64\x6f\x20\x79\x61\x20\x77\x61\x6e\x74\x20\x66" + "\x6f\x72\x20\x6e\x6f\x74\x68\x69\x6e\x67\x3f", + "\x16\x4b\x7a\x7b\xfc\xf8\x19\xe2\xe3\x95\xfb\xe7\x3b\x56\xe0\xa3\x87" + "\xbd\x64\x22\x2e\x83\x1f\xd6\x10\x27\x0c\xd7\xea\x25\x05\x54\x97\x58" + "\xbf\x75\xc0\x5a\x99\x4a\x6d\x03\x4f\x65\xf8\xf0\xe6\xfd\xca\xea\xb1" + "\xa3\x4d\x4a\x6b\x4b\x63\x6e\x07\x0a\x38\xbc\xe7\x37", + 1, 224, NULL, 0 }, + { 20, 64, 3, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" + "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", + "\xfa\x73\xb0\x08\x9d\x56\xa2\x84\xef\xb0\xf0\x75\x6c\x89\x0b\xe9\xb1" + "\xb5\xdb\xdd\x8e\xe8\x1a\x36\x55\xf8\x3e\x33\xb2\x27\x9d\x39\xbf\x3e" + "\x84\x82\x79\xa7\x22\xc8\x06\xb4\x85\xa4\x7e\x67\xc8\x07\xb9\x46\xa3" + "\x37\xbe\xe8\x94\x26\x74\x27\x88\x59\xe1\x32\x92\xfb", + 1, 400, NULL, 0 }, + { 25, 64, 4, + "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19", + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" + "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", + "\xb0\xba\x46\x56\x37\x45\x8c\x69\x90\xe5\xa8\xc5\xf6\x1d\x4a\xf7\xe5" + "\x76\xd9\x7f\xf9\x4b\x87\x2d\xe7\x6f\x80\x50\x36\x1e\xe3\xdb\xa9\x1c" + "\xa5\xc1\x1a\xa2\x5e\xb4\xd6\x79\x27\x5c\xc5\x78\x80\x63\xa5\xf1\x97" + "\x41\x12\x0c\x4f\x2d\xe2\xad\xeb\xeb\x10\xa2\x98\xdd", + 1, 400, NULL, 0 }, + { 20, 16, 5, + "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c" + "\x0c\x0c\x0c", + "\x3a\xbf\x34\xc3\x50\x3b\x2a\x23\xa4\x6e\xfc\x61\x9b\xae\xf8\x97", + "\x41\x5f\xad\x62\x71\x58\x0a\x53\x1d\x41\x79\xbc\x89\x1d\x87\xa6", + 1, 128, NULL, 0 }, + { 131, 64, 6, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x65\x73\x74\x20\x55\x73\x69\x6e\x67\x20\x4c\x61\x72\x67\x65\x72" + "\x20\x54\x68\x61\x6e\x20\x42\x6c\x6f\x63\x6b\x2d\x53\x69\x7a\x65\x20" + "\x4b\x65\x79\x20\x2d\x20\x48\x61\x73\x68\x20\x4b\x65\x79\x20\x46\x69" + "\x72\x73\x74", + "\x80\xb2\x42\x63\xc7\xc1\xa3\xeb\xb7\x14\x93\xc1\xdd\x7b\xe8\xb4\x9b" + "\x46\xd1\xf4\x1b\x4a\xee\xc1\x12\x1b\x01\x37\x83\xf8\xf3\x52\x6b\x56" + "\xd0\x37\xe0\x5f\x25\x98\xbd\x0f\xd2\x21\x5d\x6a\x1e\x52\x95\xe6\x4f" + "\x73\xf6\x3f\x0a\xec\x8b\x91\x5a\x98\x5d\x78\x65\x98", + 1, 432, NULL, 0 }, + { 131, 64, 7, + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", + "\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x74\x65\x73\x74\x20\x75\x73" + "\x69\x6e\x67\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e" + "\x20\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x6b\x65\x79\x20\x61" + "\x6e\x64\x20\x61\x20\x6c\x61\x72\x67\x65\x72\x20\x74\x68\x61\x6e\x20" + "\x62\x6c\x6f\x63\x6b\x2d\x73\x69\x7a\x65\x20\x64\x61\x74\x61\x2e\x20" + "\x54\x68\x65\x20\x6b\x65\x79\x20\x6e\x65\x65\x64\x73\x20\x74\x6f\x20" + "\x62\x65\x20\x68\x61\x73\x68\x65\x64\x20\x62\x65\x66\x6f\x72\x65\x20" + "\x62\x65\x69\x6e\x67\x20\x75\x73\x65\x64\x20\x62\x79\x20\x74\x68\x65" + "\x20\x48\x4d\x41\x43\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x2e", + "\xe3\x7b\x6a\x77\x5d\xc8\x7d\xba\xa4\xdf\xa9\xf9\x6e\x5e\x3f\xfd\xde" + "\xbd\x71\xf8\x86\x72\x89\x86\x5d\xf5\xa3\x2d\x20\xcd\xc9\x44\xb6\x02" + "\x2c\xac\x3c\x49\x82\xb1\x0d\x5e\xeb\x55\xc3\xe4\xde\x15\x13\x46\x76" + "\xfb\x6d\xe0\x44\x60\x65\xc9\x74\x40\xfa\x8c\x6a\x58", + 1, 1216, NULL, 0 }, + /* Vectors from https://csrc.nist.gov/csrc/media/projects/ */ + { 64, 32, 8, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x53\x61\x6d\x70\x6c\x65\x20\x6d\x65\x73\x73\x61\x67\x65\x20\x66\x6f" + "\x72\x20\x6b\x65\x79\x6c\x65\x6e\x3d\x62\x6c\x6f\x63\x6b\x6c\x65" + "\x6e", + "\xfc\x25\xe2\x40\x65\x8c\xa7\x85\xb7\xa8\x11\xa8\xd3\xf7\xb4\xca\x48" + "\xcf\xa2\x6a\x8a\x36\x6b\xf2\xcd\x1f\x83\x6b\x05\xfc\xb0\x24", + 1, 272, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/kasumi_test.c b/test/kat-app/kasumi_test.c similarity index 99% rename from test/kasumi_test.c rename to test/kat-app/kasumi_test.c index 6fff0c44401c0de8aa5c905132f9eabf2c8df901..ce54a96f4abfcd7e1959dcc75c73e868cd45976e 100644 --- a/test/kasumi_test.c +++ b/test/kat-app/kasumi_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -89,9 +89,10 @@ struct kasumi_test_case kasumi_f9_func_tab[] = { static int submit_kasumi_f8_jobs(struct IMB_MGR *mb_mgr, kasumi_key_sched_t **keys, - uint64_t **ivs, uint8_t **src, uint8_t **dst, - const uint32_t *bitlens, const uint32_t *bit_offsets, - int dir, const unsigned int num_jobs) + uint64_t **ivs, uint8_t ** const src, + uint8_t ** const dst, const uint32_t *bitlens, + const uint32_t *bit_offsets, const int dir, + const unsigned int num_jobs) { unsigned int i; unsigned int jobs_rx = 0; diff --git a/test/kasumi_test_vectors.h b/test/kat-app/kasumi_test_vectors.h similarity index 99% rename from test/kasumi_test_vectors.h rename to test/kat-app/kasumi_test_vectors.h index 6b989e15dcdf6b68798376cd7507a4029622dbaf..c5bffe0dd6ddda2eac05d315e2a98ab5186ed841 100644 --- a/test/kasumi_test_vectors.h +++ b/test/kat-app/kasumi_test_vectors.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2022, Intel Corporation + * Copyright (c) 2009-2023, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/test/main.c b/test/kat-app/main.c similarity index 80% rename from test/main.c rename to test/kat-app/main.c index bc61efc718a39c86ad1ba75d9d080e2751b5cdb2..a5b4d3dd2636d80802fb03b75514b01296329a98 100644 --- a/test/main.c +++ b/test/kat-app/main.c @@ -62,10 +62,13 @@ extern int chacha20_poly1305_test(struct IMB_MGR *mb_mgr); extern int null_test(struct IMB_MGR *mb_mgr); extern int snow_v_test(struct IMB_MGR *mb_mgr); extern int direct_api_param_test(struct IMB_MGR *mb_mgr); +extern int quic_ecb_test(struct IMB_MGR *mb_mgr); +extern int gmac_test(struct IMB_MGR *mb_mgr); +extern int ghash_test(struct IMB_MGR *mb_mgr); typedef int (*imb_test_t)(struct IMB_MGR *mb_mgr); -#ifdef __x86_64__ +#ifndef __aarch64__ #include "do_test.h" #endif @@ -82,7 +85,7 @@ struct imb_test { }; struct imb_test tests[] = { -#ifdef __x86_64__ +#ifndef __aarch64__ { .str = "KAT", .fn = known_answer_test, @@ -94,7 +97,7 @@ struct imb_test tests[] = { .enabled = 1 }, { - .str = "CTR", + .str = "CTR", .fn = ctr_test, .enabled = 1 }, @@ -104,7 +107,7 @@ struct imb_test tests[] = { .enabled = 1 }, { - .str = "XCBC", + .str = "XCBC", .fn = xcbc_test, .enabled = 1 }, @@ -113,6 +116,16 @@ struct imb_test tests[] = { .fn = gcm_test, .enabled = 1 }, + { + .str = "GMAC", + .fn = gmac_test, + .enabled = 1 + }, + { + .str = "GHASH", + .fn = ghash_test, + .enabled = 1 + }, { .str = "CUSTOMOP", .fn = customop_test, @@ -133,21 +146,25 @@ struct imb_test tests[] = { .fn = cmac_test, .enabled = 1 }, +#endif /* __aarch64__ */ { - .str = "zuc", + .str = "ZUC", .fn = zuc_test, .enabled = 1 }, +#ifndef __aarch64__ { .str = "KASUMI", .fn = kasumi_test, .enabled = 1 }, +#endif /* __aarch64__ */ { .str = "SNOW3G", .fn = snow3g_test, .enabled = 1 }, +#ifndef __aarch64__ { .str = "HMAC_SHA1", .fn = hmac_sha1_test, @@ -179,7 +196,7 @@ struct imb_test tests[] = { .enabled = 1 }, { - .str = "CHAINED", + .str = "CHAINED", .fn = chained_test, .enabled = 1 }, @@ -203,6 +220,7 @@ struct imb_test tests[] = { .fn = poly1305_test, .enabled = 1 }, +#endif /* __aarch64__ */ { .str = "API", .fn = api_test, @@ -218,6 +236,7 @@ struct imb_test tests[] = { .fn = clear_mem_test, .enabled = 1 }, +#ifndef __aarch64__ { .str = "CRC", .fn = crc_test, @@ -238,61 +257,76 @@ struct imb_test tests[] = { .fn = snow_v_test, .enabled = 1 }, +#endif /* __aarch64__ */ { .str = "DIRECT_API_PARAM", .fn = direct_api_param_test, .enabled = 1 }, -#endif - -#ifdef __aarch64__ - { - .str = "SNOW3G", - .fn = snow3g_test, - .enabled = 1 - }, - { - .str = "zuc", - .fn = zuc_test, - .enabled = 1 - }, +#ifndef __aarch64__ { - .str = "API", - .fn = api_test, + .str = "QUIC", + .fn = quic_ecb_test, .enabled = 1 - }, - { - .str = "DIRECT_API", - .fn = direct_api_test, - .enabled = 1 - }, - { - .str = "CLEAR_MEM", - .fn = clear_mem_test, - .enabled = 1 - }, -#endif + } +#endif /* __aarch64__ */ }; +static char *get_test_types(void) +{ + const size_t separator_length = 2; + size_t buffer_sz = 1; /* 1 for NULL termination */ + + for (unsigned i = 0; i < DIM(tests); i++) + buffer_sz += strlen(tests[i].str) + separator_length; + + char *ret_str = malloc(buffer_sz); + + if (ret_str == NULL) { + fprintf(stderr, "Error allocating memory!\n"); + exit(EXIT_FAILURE); + } + + memset(ret_str, 0, buffer_sz); + + for (unsigned i = 0; i < DIM(tests); i++) { + if (i != 0) + strcat(ret_str, ", "); + + strcat(ret_str, tests[i].str); + } + + return ret_str; +} + static void usage(const char *name) { - fprintf(stderr, + char *test_types = get_test_types(); + + fprintf(stderr, "Usage: %s [args], where args are zero or more\n" - "--test-type TEST_NAME : Run single test type\n" + "--help: Prints this page\n" + "--test-type : Run selected test type. is one of %s.\n" "--stop-on-fail: Stop test execution if a test fails\n" "--no-aesni-emu: Don't do AESNI emulation\n" "--no-avx512: Don't do AVX512\n" - "--no-avx2: Don't do AVX2\n" - "--no-avx: Don't do AVX\n" - "--no-sse: Don't do SSE\n" - "--auto-detect: auto detects current architecture " + "--no-avx2: Don't do AVX2\n" + "--no-avx: Don't do AVX\n" + "--no-sse: Don't do SSE\n" + "--no-aarch64: Don't do AARCH64\n" + "--no-sve256: Don't do SVE256\n" + "--auto-detect: Auto detects current architecture " "to run the tests\n Note: Auto detection " "option now run by default and will be removed in the future\n" - "--gfni-on: use Galois Field extensions, default: auto-detect\n" - "--gfni-off: don't use Galois Field extensions\n" - "--shani-on: use SHA extensions, default: auto-detect\n" - "--shani-off: don't use SHA extensions\n", name); + "--gfni-on: Use Galois Field extensions, default: auto-detect\n" + "--gfni-off: Don't use Galois Field extensions\n" + "--shani-on: Use SHA extensions, default: auto-detect\n" + "--shani-off: Don't use SHA extensions\n" + "--quiet: Enable quiet mode with reduced text output\n", + name, test_types); + + free(test_types); } static void @@ -314,6 +348,7 @@ print_hw_features(void) { IMB_FEATURE_VPCLMULQDQ, "VPCLMULQDQ" }, { IMB_FEATURE_GFNI, "GFNI" }, { IMB_FEATURE_AVX512_IFMA, "AVX512-IFMA" }, + { IMB_FEATURE_AVX_IFMA, "AVX-IFMA" }, { IMB_FEATURE_BMI2, "BMI2" }, { IMB_FEATURE_AARCH64, "AARCH64" }, { IMB_FEATURE_SVE256, "SVE256" }, @@ -346,24 +381,21 @@ print_hw_features(void) static unsigned check_test_string_arg(const char *param, const char *arg) { - unsigned test_idx; - - if (arg == NULL) { - fprintf(stderr, "%s requires an argument\n", param); - goto exit; + if (arg != NULL) { + for (unsigned test_idx = 0; test_idx < DIM(tests); test_idx++) + if (strcasecmp(arg, tests[test_idx].str) == 0) + return test_idx; + + /* Argument is not listed in the available options */ + fprintf(stderr, "Invalid test type \"%s\"\n", arg); + } else { + fprintf(stderr, "%s requires test type argument\n", param); } - for (test_idx = 0; test_idx < DIM(tests); test_idx++) - if (strcasecmp(arg, tests[test_idx].str) == 0) - return test_idx; + char *test_types = get_test_types(); - /* Argument is not listed in the available options */ - fprintf(stderr, "Invalid argument for %s\n", param); -exit: - fprintf(stderr, "Accepted arguments: "); - for (test_idx = 0; test_idx < DIM(tests); test_idx++) - fprintf(stderr, "%s ", tests[test_idx].str); - fprintf(stderr, "\n"); + fprintf(stderr, "Accepted test types: %s\n", test_types); + free(test_types); return DIM(tests); } @@ -383,37 +415,30 @@ int main(int argc, char **argv) { uint8_t arch_support[IMB_ARCH_NUM]; + uint8_t arch_select[IMB_ARCH_NUM]; int i, atype, auto_detect = 0; uint64_t flags = 0; int errors = 0; unsigned int stop_on_fail = 0; const uint64_t feat_flags = imb_get_feature_flags(); - /* Check version number */ - if (imb_get_version() < IMB_VERSION(0, 50, 0)) - printf("Library version detection unsupported!\n"); - else - printf("Detected library version: %s\n", imb_get_version_str()); + memset(arch_select, 0xff, sizeof(arch_select)); - /* Print available CPU features */ - print_hw_features(); - - /* Detect available architectures and features */ - if (detect_arch(arch_support) < 0) - return EXIT_FAILURE; - - for (i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0) { - usage(argv[0]); - return EXIT_SUCCESS; - } else if (update_flags_and_archs(argv[i], - arch_support, + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || + strcmp(argv[i], "--help") == 0) { + usage(argv[0]); + return EXIT_SUCCESS; + } else if (update_flags_and_archs(argv[i], + arch_select, &flags)) - continue; - else if (strcmp(argv[i], "--auto-detect") == 0) + continue; + else if (strcmp(argv[i], "--auto-detect") == 0) (void) auto_detect; /* legacy option - to be removed */ - else if (strcmp(argv[i], "--stop-on-fail") == 0) + else if (strcmp(argv[i], "--stop-on-fail") == 0) stop_on_fail = 1; + else if (strcmp(argv[i], "--quiet") == 0) + quiet_mode = 1; else if (strcmp(argv[i], "--test-type") == 0) { unsigned selected_test; @@ -432,12 +457,27 @@ main(int argc, char **argv) } i++; } - else { - usage(argv[0]); - return EXIT_FAILURE; - } } + /* Check version number */ + if (imb_get_version() < IMB_VERSION(0, 50, 0)) { + printf("Library version detection unsupported!\n"); + } else { + printf("Detected library version: %s\n", imb_get_version_str()); + printf("Tool version: %s\n", IMB_VERSION_STR); + } + + /* Print available CPU features */ + print_hw_features(); + + /* Detect available architectures and features */ + if (detect_arch(arch_support) < 0) + return EXIT_FAILURE; + + /* Combine user arch selection with HW capabilities */ + for (unsigned j = 0; j < DIM(arch_support); j++) + arch_support[j] = arch_support[j] & arch_select[j]; + /* Go through architectures */ for (atype = IMB_ARCH_NOAESNI; atype < IMB_ARCH_NUM; atype++) { IMB_MGR *p_mgr = NULL; @@ -475,11 +515,9 @@ main(int argc, char **argv) case IMB_ARCH_SVE256: init_mb_mgr_aarch64_sve256(p_mgr); break; -#endif - -#ifdef __x86_64__ - case IMB_ARCH_NOAESNI: +#else case IMB_ARCH_SSE: + case IMB_ARCH_NOAESNI: init_mb_mgr_sse(p_mgr); break; case IMB_ARCH_AVX: diff --git a/test/null_test.c b/test/kat-app/null_test.c similarity index 99% rename from test/null_test.c rename to test/kat-app/null_test.c index 3dc2d2824e95936d17380cec89497baff9a71ff2..200d07fb367821b1cf10e62e213b03a7d653a4ab 100644 --- a/test/null_test.c +++ b/test/kat-app/null_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation + Copyright (c) 2020-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/kat-app/poly1305_test.c b/test/kat-app/poly1305_test.c new file mode 100644 index 0000000000000000000000000000000000000000..a74c40a7e4d9687bce9cd72228b335d41d08596a --- /dev/null +++ b/test/kat-app/poly1305_test.c @@ -0,0 +1,231 @@ +/***************************************************************************** + Copyright (c) 2020-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "utils.h" +#include "mac_test.h" + +int poly1305_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test poly1305_test_json[]; + +static int +poly1305_job_ok(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const struct IMB_JOB *job, + const uint8_t *auth, + const uint8_t *padding, + const size_t sizeof_padding) +{ + const size_t auth_len = job->auth_tag_output_len_in_bytes; + + if (job->status != IMB_STATUS_COMPLETED) { + const int errcode = imb_get_errno(mb_mgr); + + printf("Error!: job status %d, errno %d => %s\n", + job->status, errcode, imb_get_strerror(errcode)); + return 0; + } + + /* hash checks */ + if (memcmp(padding, &auth[sizeof_padding + auth_len], + sizeof_padding)) { + printf("hash overwrite tail\n"); + hexdump(stderr, "Target", + &auth[sizeof_padding + auth_len], sizeof_padding); + return 0; + } + + if (memcmp(padding, &auth[0], sizeof_padding)) { + printf("hash overwrite head\n"); + hexdump(stderr, "Target", &auth[0], sizeof_padding); + return 0; + } + + if (memcmp((const void *) vec->tag, &auth[sizeof_padding], auth_len)) { + printf("hash mismatched\n"); + hexdump(stderr, "Received", &auth[sizeof_padding], + auth_len); + hexdump(stderr, "Expected", (const void *) vec->tag, + auth_len); + return 0; + } + return 1; +} + +static int +test_poly1305(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const int dir, + const int num_jobs) +{ + struct IMB_JOB *job; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + int i = 0, jobs_rx = 0, ret = -1; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + auths[i] = malloc(16 + (sizeof(padding) * 2)); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + + memset(auths[i], -1, 16 + (sizeof(padding) * 2)); + } + + + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + /** + * Submit all jobs then flush any outstanding jobs + */ + for (i = 0; i < num_jobs; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_POLY1305; + + job->u.POLY1305._key = vec->key; + job->src = (const void *) vec->msg; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = 16; + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job) { + jobs_rx++; + if (!poly1305_job_ok(mb_mgr, vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + jobs_rx++; + + if (!poly1305_job_ok(mb_mgr, vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + + + ret = 0; + + end: + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static void +test_poly1305_vectors(struct IMB_MGR *mb_mgr, + const int num_jobs, + struct test_suite_context *ctx, + const char *banner) +{ + const struct mac_test *v = poly1305_test_json; + + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); + for (; v->msg != NULL; v++) { + if (!quiet_mode) { +#ifdef DEBUG + printf("RFC7539 Test Case %zu key_len:%zu " + "data_len:%zu digest_len:%zu\n", + v->tcId, + v->keySize, + v->msgSize / 8, + v->tagSize); +#else + printf("."); +#endif + } + + if (test_poly1305(mb_mgr, v, IMB_DIR_ENCRYPT, num_jobs)) { + printf("error #%zu encrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else + test_suite_update(ctx, 1, 0); + + if (test_poly1305(mb_mgr, v, IMB_DIR_DECRYPT, num_jobs)) { + printf("error #%zu decrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else + test_suite_update(ctx, 1, 0); + } + if (!quiet_mode) + printf("\n"); +} + + +int +poly1305_test(struct IMB_MGR *mb_mgr) +{ + struct test_suite_context ctx; + int i, errors; + + test_suite_start(&ctx, "POLY1305"); + for (i = 1; i < 20; i++) + test_poly1305_vectors(mb_mgr, i, &ctx, "Poly1305 RFC7539 vectors"); + errors = test_suite_end(&ctx); + + return errors; +} diff --git a/test/kat-app/poly1305_test.json.c b/test/kat-app/poly1305_test.json.c new file mode 100644 index 0000000000000000000000000000000000000000..1a479108fb3c8bbb99e79bb99f23be0037aa0480 --- /dev/null +++ b/test/kat-app/poly1305_test.json.c @@ -0,0 +1,139 @@ +/* POLY1305, RFC7539 */ +#include "mac_test.h" + +const struct mac_test poly1305_test_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc7539 */ + {256, 128, 1, + "\x85\xd6\xbe\x78\x57\x55\x6d\x33\x7f\x44\x52\xfe\x42\xd5\x06\xa8\x01\x03" + "\x80\x8a\xfb\x0d\xb2\xfd\x4a\xbf\xf6\xaf\x41\x49\xf5\x1b", + "\x43\x72\x79\x70\x74\x6f\x67\x72\x61\x70\x68\x69\x63\x20\x46\x6f\x72\x75" + "\x6d\x20\x52\x65\x73\x65\x61\x72\x63\x68\x20\x47\x72\x6f\x75\x70", + "\xa8\x06\x1d\xc1\x30\x51\x36\xc6\xc2\x2b\x8b\xaf\x0c\x01\x27\xa9", 1, 272, + NULL, 0}, + {256, 128, 2, + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, 512, + NULL, 0}, + {256, 128, 3, + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x36\xe5" + "\xf6\xb5\xc5\xe0\x60\x70\xf0\xef\xca\x96\x22\x7a\x86\x3e", + "\x41\x6e\x79\x20\x73\x75\x62\x6d\x69\x73\x73\x69\x6f\x6e\x20\x74\x6f\x20" + "\x74\x68\x65\x20\x49\x45\x54\x46\x20\x69\x6e\x74\x65\x6e\x64\x65\x64\x20" + "\x62\x79\x20\x74\x68\x65\x20\x43\x6f\x6e\x74\x72\x69\x62\x75\x74\x6f\x72" + "\x20\x66\x6f\x72\x20\x70\x75\x62\x6c\x69\x63\x61\x74\x69\x6f\x6e\x20\x61" + "\x73\x20\x61\x6c\x6c\x20\x6f\x72\x20\x70\x61\x72\x74\x20\x6f\x66\x20\x61" + "\x6e\x20\x49\x45\x54\x46\x20\x49\x6e\x74\x65\x72\x6e\x65\x74\x2d\x44\x72" + "\x61\x66\x74\x20\x6f\x72\x20\x52\x46\x43\x20\x61\x6e\x64\x20\x61\x6e\x79" + "\x20\x73\x74\x61\x74\x65\x6d\x65\x6e\x74\x20\x6d\x61\x64\x65\x20\x77\x69" + "\x74\x68\x69\x6e\x20\x74\x68\x65\x20\x63\x6f\x6e\x74\x65\x78\x74\x20\x6f" + "\x66\x20\x61\x6e\x20\x49\x45\x54\x46\x20\x61\x63\x74\x69\x76\x69\x74\x79" + "\x20\x69\x73\x20\x63\x6f\x6e\x73\x69\x64\x65\x72\x65\x64\x20\x61\x6e\x20" + "\x22\x49\x45\x54\x46\x20\x43\x6f\x6e\x74\x72\x69\x62\x75\x74\x69\x6f\x6e" + "\x22\x2e\x20\x53\x75\x63\x68\x20\x73\x74\x61\x74\x65\x6d\x65\x6e\x74\x73" + "\x20\x69\x6e\x63\x6c\x75\x64\x65\x20\x6f\x72\x61\x6c\x20\x73\x74\x61\x74" + "\x65\x6d\x65\x6e\x74\x73\x20\x69\x6e\x20\x49\x45\x54\x46\x20\x73\x65\x73" + "\x73\x69\x6f\x6e\x73\x2c\x20\x61\x73\x20\x77\x65\x6c\x6c\x20\x61\x73\x20" + "\x77\x72\x69\x74\x74\x65\x6e\x20\x61\x6e\x64\x20\x65\x6c\x65\x63\x74\x72" + "\x6f\x6e\x69\x63\x20\x63\x6f\x6d\x6d\x75\x6e\x69\x63\x61\x74\x69\x6f\x6e" + "\x73\x20\x6d\x61\x64\x65\x20\x61\x74\x20\x61\x6e\x79\x20\x74\x69\x6d\x65" + "\x20\x6f\x72\x20\x70\x6c\x61\x63\x65\x2c\x20\x77\x68\x69\x63\x68\x20\x61" + "\x72\x65\x20\x61\x64\x64\x72\x65\x73\x73\x65\x64\x20\x74\x6f", + "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70\xf0\xef\xca\x96\x22\x7a\x86\x3e", 1, + 3000, NULL, 0}, + {256, 128, 4, + "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70\xf0\xef\xca\x96\x22\x7a\x86\x3e\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x41\x6e\x79\x20\x73\x75\x62\x6d\x69\x73\x73\x69\x6f\x6e\x20\x74\x6f\x20" + "\x74\x68\x65\x20\x49\x45\x54\x46\x20\x69\x6e\x74\x65\x6e\x64\x65\x64\x20" + "\x62\x79\x20\x74\x68\x65\x20\x43\x6f\x6e\x74\x72\x69\x62\x75\x74\x6f\x72" + "\x20\x66\x6f\x72\x20\x70\x75\x62\x6c\x69\x63\x61\x74\x69\x6f\x6e\x20\x61" + "\x73\x20\x61\x6c\x6c\x20\x6f\x72\x20\x70\x61\x72\x74\x20\x6f\x66\x20\x61" + "\x6e\x20\x49\x45\x54\x46\x20\x49\x6e\x74\x65\x72\x6e\x65\x74\x2d\x44\x72" + "\x61\x66\x74\x20\x6f\x72\x20\x52\x46\x43\x20\x61\x6e\x64\x20\x61\x6e\x79" + "\x20\x73\x74\x61\x74\x65\x6d\x65\x6e\x74\x20\x6d\x61\x64\x65\x20\x77\x69" + "\x74\x68\x69\x6e\x20\x74\x68\x65\x20\x63\x6f\x6e\x74\x65\x78\x74\x20\x6f" + "\x66\x20\x61\x6e\x20\x49\x45\x54\x46\x20\x61\x63\x74\x69\x76\x69\x74\x79" + "\x20\x69\x73\x20\x63\x6f\x6e\x73\x69\x64\x65\x72\x65\x64\x20\x61\x6e\x20" + "\x22\x49\x45\x54\x46\x20\x43\x6f\x6e\x74\x72\x69\x62\x75\x74\x69\x6f\x6e" + "\x22\x2e\x20\x53\x75\x63\x68\x20\x73\x74\x61\x74\x65\x6d\x65\x6e\x74\x73" + "\x20\x69\x6e\x63\x6c\x75\x64\x65\x20\x6f\x72\x61\x6c\x20\x73\x74\x61\x74" + "\x65\x6d\x65\x6e\x74\x73\x20\x69\x6e\x20\x49\x45\x54\x46\x20\x73\x65\x73" + "\x73\x69\x6f\x6e\x73\x2c\x20\x61\x73\x20\x77\x65\x6c\x6c\x20\x61\x73\x20" + "\x77\x72\x69\x74\x74\x65\x6e\x20\x61\x6e\x64\x20\x65\x6c\x65\x63\x74\x72" + "\x6f\x6e\x69\x63\x20\x63\x6f\x6d\x6d\x75\x6e\x69\x63\x61\x74\x69\x6f\x6e" + "\x73\x20\x6d\x61\x64\x65\x20\x61\x74\x20\x61\x6e\x79\x20\x74\x69\x6d\x65" + "\x20\x6f\x72\x20\x70\x6c\x61\x63\x65\x2c\x20\x77\x68\x69\x63\x68\x20\x61" + "\x72\x65\x20\x61\x64\x64\x72\x65\x73\x73\x65\x64\x20\x74\x6f", + "\xf3\x47\x7e\x7c\xd9\x54\x17\xaf\x89\xa6\xb8\x79\x4c\x31\x0c\xf0", 1, + 3000, NULL, 0}, + {256, 128, 5, + "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a\xf3\x33\x88\x86\x04\xf6\xb5\xf0\x47\x39" + "\x17\xc1\x40\x2b\x80\x09\x9d\xca\x5c\xbc\x20\x70\x75\xc0", + "\x27\x54\x77\x61\x73\x20\x62\x72\x69\x6c\x6c\x69\x67\x2c\x20\x61\x6e\x64" + "\x20\x74\x68\x65\x20\x73\x6c\x69\x74\x68\x79\x20\x74\x6f\x76\x65\x73\x0a" + "\x44\x69\x64\x20\x67\x79\x72\x65\x20\x61\x6e\x64\x20\x67\x69\x6d\x62\x6c" + "\x65\x20\x69\x6e\x20\x74\x68\x65\x20\x77\x61\x62\x65\x3a\x0a\x41\x6c\x6c" + "\x20\x6d\x69\x6d\x73\x79\x20\x77\x65\x72\x65\x20\x74\x68\x65\x20\x62\x6f" + "\x72\x6f\x67\x6f\x76\x65\x73\x2c\x0a\x41\x6e\x64\x20\x74\x68\x65\x20\x6d" + "\x6f\x6d\x65\x20\x72\x61\x74\x68\x73\x20\x6f\x75\x74\x67\x72\x61\x62\x65" + "\x2e", + "\x45\x41\x66\x9a\x7e\xaa\xee\x61\xe7\x08\xdc\x7c\xbc\xc5\xeb\x62", 1, + 1016, NULL, 0}, + {256, 128, 6, + "\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, 128, + NULL, 0}, + {256, 128, 7, + "\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, 128, + NULL, 0}, + {256, 128, 8, + "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x11\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, 384, + NULL, 0}, + {256, 128, 9, + "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfb\xfe" + "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\x01\x01\x01\x01" + "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, 384, + NULL, 0}, + {256, 128, 10, + "\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xfd\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xfa\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, 128, + NULL, 0}, + {256, 128, 11, + "\x01\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xe3\x35\x94\xd7\x50\x5e\x43\xb9\x00\x00\x00\x00\x00\x00\x00\x00\x33\x94" + "\xd7\x50\x5e\x43\x79\xcd\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x14\x00\x00\x00\x00\x00\x00\x00\x55\x00\x00\x00\x00\x00\x00\x00", 1, 512, + NULL, 0}, + {256, 128, 12, + "\x01\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xe3\x35\x94\xd7\x50\x5e\x43\xb9\x00\x00\x00\x00\x00\x00\x00\x00\x33\x94" + "\xd7\x50\x5e\x43\x79\xcd\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, 384, + NULL, 0}, + {0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0} +}; diff --git a/test/pon_test.c b/test/kat-app/pon_test.c similarity index 97% rename from test/pon_test.c rename to test/kat-app/pon_test.c index 524035ef758d5b289e0347acb026261da6815631..20a6319a20ec2f1b9dcba19323af1a8e34ed2618 100644 --- a/test/pon_test.c +++ b/test/kat-app/pon_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -526,8 +526,10 @@ test_pon(struct IMB_MGR *mb_mgr, crc_output = (uint32_t) (tag_output >> 32); #ifdef DEBUG - printf("CRC received 0x%08x\n", crc_output); - printf("BIP received 0x%08x\n", bip_output); + if (!quiet_mode) { + printf("CRC received 0x%08x\n", crc_output); + printf("BIP received 0x%08x\n", bip_output); + } #endif #ifdef DEBUG @@ -640,14 +642,16 @@ test_pon_std_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx) printf("PON (AES128-CTR/CRC/BIP) test vectors:\n"); for (vect = 0; vect < vectors_cnt; vect++) { + if (!quiet_mode) { #ifdef DEBUG - printf("Vector %d/%d CIPHLen:%d BIPLen:%d\n", - vect + 1, vectors_cnt, - (int) pon_vectors[vect].length_to_cipher, - (int) pon_vectors[vect].length_to_bip); + printf("Vector %d/%d CIPHLen:%d BIPLen:%d\n", + vect + 1, vectors_cnt, + (int) pon_vectors[vect].length_to_cipher, + (int) pon_vectors[vect].length_to_bip); #else - printf("."); + printf("."); #endif + } if (pon_vectors[vect].key != NULL) IMB_AES_KEYEXP_128(mb_mgr, pon_vectors[vect].key, @@ -683,7 +687,8 @@ test_pon_std_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx) } else test_suite_update(ctx, 1, 0); } - printf("\n"); + if (!quiet_mode) + printf("\n"); return errors; } diff --git a/test/kat-app/quic_ecb_test.c b/test/kat-app/quic_ecb_test.c new file mode 100644 index 0000000000000000000000000000000000000000..c7c5e84306df8a21b18970c74f71f635c228a475 --- /dev/null +++ b/test/kat-app/quic_ecb_test.c @@ -0,0 +1,264 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include +#include + +#include + +#include "utils.h" + +int quic_ecb_test(struct IMB_MGR *mb_mgr); + +struct quic_ecb_vector { + const uint8_t *K; /* key */ + const uint8_t *P; /* plain text (16 bytes) */ + const uint8_t *C; /* cipher text - same length as plain text */ + uint32_t Klen; /* key length */ +}; + +/* 128-bit */ +static const uint8_t ecb_128_K1[] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c +}; +static const uint8_t ecb_128_P1[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a +}; +static const uint8_t ecb_128_C1[] = { + 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60, + 0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97, +}; + +/* 256-bit */ +static const uint8_t ecb_256_K20[] = { + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, + 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, + 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 +}; +static const uint8_t ecb_256_P20[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, +}; +static const uint8_t ecb_256_P21[] = { + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, +}; +static const uint8_t ecb_256_P22[] = { + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, +}; +static const uint8_t ecb_256_P23[] = { + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; +static const uint8_t ecb_256_C20[] = { + 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c, + 0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8, +}; +static const uint8_t ecb_256_C21[] = { + 0x59, 0x1c, 0xcb, 0x10, 0xd4, 0x10, 0xed, 0x26, + 0xdc, 0x5b, 0xa7, 0x4a, 0x31, 0x36, 0x28, 0x70, +}; +static const uint8_t ecb_256_C22[] = { + 0xb6, 0xed, 0x21, 0xb9, 0x9c, 0xa6, 0xf4, 0xf9, + 0xf1, 0x53, 0xe7, 0xb1, 0xbe, 0xaf, 0xed, 0x1d, +}; +static const uint8_t ecb_256_C23[] = { + 0x23, 0x30, 0x4b, 0x7a, 0x39, 0xf9, 0xf3, 0xff, + 0x06, 0x7d, 0x8d, 0x8f, 0x9e, 0x24, 0xec, 0xc7 +}; + +static const struct quic_ecb_vector quic_ecb_vectors[] = { + {ecb_128_K1, ecb_128_P1, ecb_128_C1, sizeof(ecb_128_K1)}, + {ecb_256_K20, ecb_256_P20, ecb_256_C20, sizeof(ecb_256_K20)}, + {ecb_256_K20, ecb_256_P21, ecb_256_C21, sizeof(ecb_256_K20)}, + {ecb_256_K20, ecb_256_P22, ecb_256_C22, sizeof(ecb_256_K20)}, + {ecb_256_K20, ecb_256_P23, ecb_256_C23, sizeof(ecb_256_K20)}, +}; + +static int +test_quic_ecb_many(struct IMB_MGR *mb_mgr, + void *enc_keys, + const uint8_t *in_text, + const uint8_t *out_text, + const int in_place, + const int key_len, + const int num_jobs) +{ + const unsigned text_len = 16; + const unsigned out_len = 5; + uint8_t **src_bufs = malloc(num_jobs * sizeof(void *)); + uint8_t **dst_bufs = malloc(num_jobs * sizeof(void *)); + int i, ret = -1; + + for (i = 0; i < num_jobs; i++) { + src_bufs[i] = malloc(text_len); + memcpy(src_bufs[i], in_text, text_len); + + dst_bufs[i] = malloc(out_len); + memset(dst_bufs[i], -1, out_len); + } + + if (in_place) { + imb_quic_hp_aes_ecb(mb_mgr, enc_keys, + (void **) src_bufs, + (const void * const*) src_bufs, + num_jobs, key_len); + } else { + imb_quic_hp_aes_ecb(mb_mgr, enc_keys, + (void **) dst_bufs, + (const void * const*) src_bufs, + num_jobs, key_len); + } + + const int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("QUIC AES-ECB error status:%d, %s\n", err, + imb_get_strerror(err)); + goto end; + } + + for (i = 0; i < num_jobs; i++) { + const uint8_t *d = (in_place) ? src_bufs[i] : dst_bufs[i]; + + if (memcmp(d, out_text, out_len) != 0) { + printf("QUIC AES-ECB %d vector mismatched\n", i); + hexdump(stderr, "Expected", + out_text, out_len); + hexdump(stderr, "Received", + d, out_len); + goto end; + } + } + + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + free(src_bufs[i]); + free(dst_bufs[i]); + } + free(src_bufs); + free(dst_bufs); + return ret; +} + +static void +test_quic_ecb_vectors(struct IMB_MGR *mb_mgr, const int vec_cnt, + const struct quic_ecb_vector *vec_tab, const char *banner, + const int num_jobs, + struct test_suite_context *ts128, + struct test_suite_context *ts256) +{ + int vect; + DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); + DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); + + if (!quiet_mode) + printf("%s (N jobs = %d):\n", banner, num_jobs); + for (vect = 0; vect < vec_cnt; vect++) { + struct test_suite_context *ctx = NULL; + + if (!quiet_mode) { +#ifdef DEBUG + printf("[%d/%d] Standard vector key_len:%d\n", + vect + 1, vec_cnt, + (int) vec_tab[vect].Klen); +#else + printf("."); +#endif + } + + switch (vec_tab[vect].Klen) { + case 16: + IMB_AES_KEYEXP_128(mb_mgr, vec_tab[vect].K, enc_keys, + dec_keys); + ctx = ts128; + break; + case 32: + default: + IMB_AES_KEYEXP_256(mb_mgr, vec_tab[vect].K, enc_keys, + dec_keys); + ctx = ts256; + break; + } + + if (test_quic_ecb_many(mb_mgr, enc_keys, + vec_tab[vect].P, vec_tab[vect].C, 0, + vec_tab[vect].Klen, num_jobs)) { + printf("error #%d encrypt\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_quic_ecb_many(mb_mgr, enc_keys, + vec_tab[vect].P, vec_tab[vect].C, 1, + vec_tab[vect].Klen, num_jobs)) { + printf("error #%d encrypt in-place\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + } + + if (!quiet_mode) + printf("\n"); +} + +int +quic_ecb_test(struct IMB_MGR *mb_mgr) +{ + struct test_suite_context ts128, ts256; + const int num_jobs_tab[] = { + 1, 3, 4, 5, 7, 8, 9, 15, 16, 17 + }; + unsigned i; + int errors = 0; + + test_suite_start(&ts128, "QUIC-HP-AES-ECB-128"); + test_suite_start(&ts256, "QUIC-HP-AES-ECB-256"); + + for (i = 0; i < DIM(num_jobs_tab); i++) + test_quic_ecb_vectors(mb_mgr, DIM(quic_ecb_vectors), + quic_ecb_vectors, + "QUIC-HP-AES-ECB test vectors", + num_jobs_tab[i], + &ts128, &ts256); + + errors = test_suite_end(&ts128); + errors += test_suite_end(&ts256); + + return errors; +} diff --git a/test/kat-app/sha_test.c b/test/kat-app/sha_test.c new file mode 100644 index 0000000000000000000000000000000000000000..bc3bb5800f62e07122017081ad89a9a9f6d869bd --- /dev/null +++ b/test/kat-app/sha_test.c @@ -0,0 +1,272 @@ +/***************************************************************************** + Copyright (c) 2018-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "gcm_ctr_vectors_test.h" +#include "utils.h" +#include "mac_test.h" + +int sha_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test sha_test_json[]; + +static int +sha_job_ok(const struct mac_test *vec, + const struct IMB_JOB *job, + const uint8_t *auth, + const uint8_t *padding, + const size_t sizeof_padding) +{ + if (job->status != IMB_STATUS_COMPLETED) { + printf("line:%d job error status:%d ", __LINE__, job->status); + return 0; + } + + /* hash checks */ + if (memcmp(padding, &auth[sizeof_padding + (vec->tagSize / 8)], + sizeof_padding)) { + printf("hash overwrite tail\n"); + hexdump(stderr, "Target", + &auth[sizeof_padding + (vec->tagSize / 8)], + sizeof_padding); + return 0; + } + + if (memcmp(padding, &auth[0], sizeof_padding)) { + printf("hash overwrite head\n"); + hexdump(stderr, "Target", &auth[0], sizeof_padding); + return 0; + } + + if (memcmp((const void *) vec->tag, &auth[sizeof_padding], + vec->tagSize / 8)) { + printf("hash mismatched\n"); + hexdump(stderr, "Received", &auth[sizeof_padding], + vec->tagSize / 8); + hexdump(stderr, "Expected", (const void *) vec->tag, + vec->tagSize / 8); + return 0; + } + return 1; +} + +static int +test_sha(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const int num_jobs, + const int sha_type) +{ + struct IMB_JOB *job; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + int i = 0, jobs_rx = 0, ret = -1; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->tagSize / 8 + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + /* empty the manager */ + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + + memset(job, 0, sizeof(*job)); + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize / 8; + job->src = (const void *) vec->msg; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->cipher_mode = IMB_CIPHER_NULL; + switch (sha_type) { + case 1: + job->hash_alg = IMB_AUTH_SHA_1; + break; + case 224: + job->hash_alg = IMB_AUTH_SHA_224; + break; + case 256: + job->hash_alg = IMB_AUTH_SHA_256; + break; + case 384: + job->hash_alg = IMB_AUTH_SHA_384; + break; + case 512: + default: + job->hash_alg = IMB_AUTH_SHA_512; + break; + } + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job) { + jobs_rx++; + if (!sha_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + jobs_rx++; + if (!sha_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + /* empty the manager before next tests */ + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static void +test_sha_vectors(struct IMB_MGR *mb_mgr, + struct test_suite_context *sha1_ctx, + struct test_suite_context *sha224_ctx, + struct test_suite_context *sha256_ctx, + struct test_suite_context *sha384_ctx, + struct test_suite_context *sha512_ctx, + const int num_jobs) +{ + struct test_suite_context *ctx; + const struct mac_test *v = sha_test_json; + int sha_type; + + if (!quiet_mode) + printf("SHA standard test vectors (N jobs = %d):\n", num_jobs); + for (; v->msg != NULL; v++) { + + switch (v->tagSize) { + case 160: + ctx = sha1_ctx; + sha_type = 1; + break; + case 224: + ctx = sha224_ctx; + sha_type = 224; + break; + case 256: + ctx = sha256_ctx; + sha_type = 256; + break; + case 384: + ctx = sha384_ctx; + sha_type = 384; + break; + case 512: + ctx = sha512_ctx; + sha_type = 512; + break; + default: + ctx = sha1_ctx; + printf("error #%zu, invalid tag size\n", v->tcId); + test_suite_update(ctx, 0, 1); + continue; + } +#ifdef DEBUG + if (!quiet_mode) { + printf("SHA%d Test Case %zu " + "data_len:%zu digest_len:%zu\n", + sha_type, v->tcId, v->msgSize / 8, v->tagSize / 8); + } +#endif + if (test_sha(mb_mgr, v, num_jobs, sha_type)) { + printf("error #%zu\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + } +} + +int +sha_test(struct IMB_MGR *mb_mgr) +{ + struct test_suite_context sha1_ctx, sha224_ctx, sha256_ctx; + struct test_suite_context sha384_ctx, sha512_ctx; + int errors; + unsigned i; + + test_suite_start(&sha1_ctx, "SHA1"); + test_suite_start(&sha224_ctx, "SHA224"); + test_suite_start(&sha256_ctx, "SHA256"); + test_suite_start(&sha384_ctx, "SHA384"); + test_suite_start(&sha512_ctx, "SHA512"); + for (i = 1; i <= 17; i++) { + test_sha_vectors(mb_mgr, &sha1_ctx, &sha224_ctx, + &sha256_ctx, &sha384_ctx, &sha512_ctx, i); + } + errors = test_suite_end(&sha1_ctx); + errors += test_suite_end(&sha224_ctx); + errors += test_suite_end(&sha256_ctx); + errors += test_suite_end(&sha384_ctx); + errors += test_suite_end(&sha512_ctx); + + return errors; +} diff --git a/test/kat-app/sha_test.json.c b/test/kat-app/sha_test.json.c new file mode 100644 index 0000000000000000000000000000000000000000..27526734746f9407c340626206193abb251132eb --- /dev/null +++ b/test/kat-app/sha_test.json.c @@ -0,0 +1,184 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* SHA1, SHA2 */ +#include "mac_test.h" + +const struct mac_test sha_test_json[] = { + /* + * Vectors from https://csrc.nist.gov/csrc/media/projects/ + * cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + */ + { 0, 160, 1, NULL, "\x61\x62\x63", + "\xa9\x99\x3e\x36\x47\x06\x81\x6a\xba\x3e\x25\x71\x78\x50\xc2\x6c\x9c\xd0" + "\xd8\x9d", + 1, 24, NULL, 0 }, + { 0, 224, 2, NULL, "\x61\x62\x63", + "\x23\x09\x7d\x22\x34\x05\xd8\x22\x86\x42\xa4\x77\xbd\xa2\x55\xb3\x2a\xad" + "\xbc\xe4\xbd\xa0\xb3\xf7\xe3\x6c\x9d\xa7", + 1, 24, NULL, 0 }, + { 0, 256, 3, NULL, "\x61\x62\x63", + "\xba\x78\x16\xbf\x8f\x01\xcf\xea\x41\x41\x40\xde\x5d\xae\x22\x23\xb0\x03" + "\x61\xa3\x96\x17\x7a\x9c\xb4\x10\xff\x61\xf2\x00\x15\xad", + 1, 24, NULL, 0 }, + { 0, 384, 4, NULL, "\x61\x62\x63", + "\xcb\x00\x75\x3f\x45\xa3\x5e\x8b\xb5\xa0\x3d\x69\x9a\xc6\x50\x07\x27\x2c" + "\x32\xab\x0e\xde\xd1\x63\x1a\x8b\x60\x5a\x43\xff\x5b\xed\x80\x86\x07\x2b" + "\xa1\xe7\xcc\x23\x58\xba\xec\xa1\x34\xc8\x25\xa7", + 1, 24, NULL, 0 }, + { 0, 512, 5, NULL, "\x61\x62\x63", + "\xdd\xaf\x35\xa1\x93\x61\x7a\xba\xcc\x41\x73\x49\xae\x20\x41\x31\x12\xe6" + "\xfa\x4e\x89\xa9\x7e\xa2\x0a\x9e\xee\xe6\x4b\x55\xd3\x9a\x21\x92\x99\x2a" + "\x27\x4f\xc1\xa8\x36\xba\x3c\x23\xa3\xfe\xeb\xbd\x45\x4d\x44\x23\x64\x3c" + "\xe8\x0e\x2a\x9a\xc9\x4f\xa5\x4c\xa4\x9f", + 1, 24, NULL, 0 }, + { 0, 160, 6, NULL, "", + "\xda\x39\xa3\xee\x5e\x6b\x4b\x0d\x32\x55\xbf\xef\x95\x60\x18\x90\xaf\xd8" + "\x07\x09", + 1, 0, NULL, 0 }, + { 0, 224, 7, NULL, "", + "\xd1\x4a\x02\x8c\x2a\x3a\x2b\xc9\x47\x61\x02\xbb\x28\x82\x34\xc4\x15\xa2" + "\xb0\x1f\x82\x8e\xa6\x2a\xc5\xb3\xe4\x2f", + 1, 0, NULL, 0 }, + { 0, 256, 8, NULL, "", + "\xe3\xb0\xc4\x42\x98\xfc\x1c\x14\x9a\xfb\xf4\xc8\x99\x6f\xb9\x24\x27\xae" + "\x41\xe4\x64\x9b\x93\x4c\xa4\x95\x99\x1b\x78\x52\xb8\x55", + 1, 0, NULL, 0 }, + { 0, 384, 9, NULL, "", + "\x38\xb0\x60\xa7\x51\xac\x96\x38\x4c\xd9\x32\x7e\xb1\xb1\xe3\x6a\x21\xfd" + "\xb7\x11\x14\xbe\x07\x43\x4c\x0c\xc7\xbf\x63\xf6\xe1\xda\x27\x4e\xde\xbf" + "\xe7\x6f\x65\xfb\xd5\x1a\xd2\xf1\x48\x98\xb9\x5b", + 1, 0, NULL, 0 }, + { 0, 512, 10, NULL, "", + "\xcf\x83\xe1\x35\x7e\xef\xb8\xbd\xf1\x54\x28\x50\xd6\x6d\x80\x07\xd6\x20" + "\xe4\x05\x0b\x57\x15\xdc\x83\xf4\xa9\x21\xd3\x6c\xe9\xce\x47\xd0\xd1\x3c" + "\x5d\x85\xf2\xb0\xff\x83\x18\xd2\x87\x7e\xec\x2f\x63\xb9\x31\xbd\x47\x41" + "\x7a\x81\xa5\x38\x32\x7a\xf9\x27\xda\x3e", + 1, 0, NULL, 0 }, + { 0, 160, 11, NULL, + "\x61\x62\x63\x64\x62\x63\x64\x65\x63\x64\x65\x66\x64\x65\x66\x67\x65\x66" + "\x67\x68\x66\x67\x68\x69\x67\x68\x69\x6a\x68\x69\x6a\x6b\x69\x6a\x6b\x6c" + "\x6a\x6b\x6c\x6d\x6b\x6c\x6d\x6e\x6c\x6d\x6e\x6f\x6d\x6e\x6f\x70\x6e\x6f" + "\x70\x71", + "\x84\x98\x3e\x44\x1c\x3b\xd2\x6e\xba\xae\x4a\xa1\xf9\x51\x29\xe5\xe5\x46" + "\x70\xf1", + 1, 448, NULL, 0 }, + { 0, 224, 12, NULL, + "\x61\x62\x63\x64\x62\x63\x64\x65\x63\x64\x65\x66\x64\x65\x66\x67\x65\x66" + "\x67\x68\x66\x67\x68\x69\x67\x68\x69\x6a\x68\x69\x6a\x6b\x69\x6a\x6b\x6c" + "\x6a\x6b\x6c\x6d\x6b\x6c\x6d\x6e\x6c\x6d\x6e\x6f\x6d\x6e\x6f\x70\x6e\x6f" + "\x70\x71", + "\x75\x38\x8b\x16\x51\x27\x76\xcc\x5d\xba\x5d\xa1\xfd\x89\x01\x50\xb0\xc6" + "\x45\x5c\xb4\xf5\x8b\x19\x52\x52\x25\x25", + 1, 448, NULL, 0 }, + { 0, 256, 13, NULL, + "\x61\x62\x63\x64\x62\x63\x64\x65\x63\x64\x65\x66\x64\x65\x66\x67\x65\x66" + "\x67\x68\x66\x67\x68\x69\x67\x68\x69\x6a\x68\x69\x6a\x6b\x69\x6a\x6b\x6c" + "\x6a\x6b\x6c\x6d\x6b\x6c\x6d\x6e\x6c\x6d\x6e\x6f\x6d\x6e\x6f\x70\x6e\x6f" + "\x70\x71", + "\x24\x8d\x6a\x61\xd2\x06\x38\xb8\xe5\xc0\x26\x93\x0c\x3e\x60\x39\xa3\x3c" + "\xe4\x59\x64\xff\x21\x67\xf6\xec\xed\xd4\x19\xdb\x06\xc1", + 1, 448, NULL, 0 }, + { 0, 384, 14, NULL, + "\x61\x62\x63\x64\x62\x63\x64\x65\x63\x64\x65\x66\x64\x65\x66\x67\x65\x66" + "\x67\x68\x66\x67\x68\x69\x67\x68\x69\x6a\x68\x69\x6a\x6b\x69\x6a\x6b\x6c" + "\x6a\x6b\x6c\x6d\x6b\x6c\x6d\x6e\x6c\x6d\x6e\x6f\x6d\x6e\x6f\x70\x6e\x6f" + "\x70\x71", + "\x33\x91\xfd\xdd\xfc\x8d\xc7\x39\x37\x07\xa6\x5b\x1b\x47\x09\x39\x7c\xf8" + "\xb1\xd1\x62\xaf\x05\xab\xfe\x8f\x45\x0d\xe5\xf3\x6b\xc6\xb0\x45\x5a\x85" + "\x20\xbc\x4e\x6f\x5f\xe9\x5b\x1f\xe3\xc8\x45\x2b", + 1, 448, NULL, 0 }, + { 0, 512, 15, NULL, + "\x61\x62\x63\x64\x62\x63\x64\x65\x63\x64\x65\x66\x64\x65\x66\x67\x65\x66" + "\x67\x68\x66\x67\x68\x69\x67\x68\x69\x6a\x68\x69\x6a\x6b\x69\x6a\x6b\x6c" + "\x6a\x6b\x6c\x6d\x6b\x6c\x6d\x6e\x6c\x6d\x6e\x6f\x6d\x6e\x6f\x70\x6e\x6f" + "\x70\x71", + "\x20\x4a\x8f\xc6\xdd\xa8\x2f\x0a\x0c\xed\x7b\xeb\x8e\x08\xa4\x16\x57\xc1" + "\x6e\xf4\x68\xb2\x28\xa8\x27\x9b\xe3\x31\xa7\x03\xc3\x35\x96\xfd\x15\xc1" + "\x3b\x1b\x07\xf9\xaa\x1d\x3b\xea\x57\x78\x9c\xa0\x31\xad\x85\xc7\xa7\x1d" + "\xd7\x03\x54\xec\x63\x12\x38\xca\x34\x45", + 1, 448, NULL, 0 }, + { 0, 160, 16, NULL, + "\x61\x62\x63\x64\x65\x66\x67\x68\x62\x63\x64\x65\x66\x67\x68\x69\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x64\x65\x66\x67\x68\x69\x6a\x6b\x65\x66\x67\x68" + "\x69\x6a\x6b\x6c\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70" + "\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x6c\x6d" + "\x6e\x6f\x70\x71\x72\x73\x6d\x6e\x6f\x70\x71\x72\x73\x74\x6e\x6f\x70\x71" + "\x72\x73\x74\x75", + "\xa4\x9b\x24\x46\xa0\x2c\x64\x5b\xf4\x19\xf9\x95\xb6\x70\x91\x25\x3a\x04" + "\xa2\x59", + 1, 896, NULL, 0 }, + { 0, 224, 17, NULL, + "\x61\x62\x63\x64\x65\x66\x67\x68\x62\x63\x64\x65\x66\x67\x68\x69\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x64\x65\x66\x67\x68\x69\x6a\x6b\x65\x66\x67\x68" + "\x69\x6a\x6b\x6c\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70" + "\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x6c\x6d" + "\x6e\x6f\x70\x71\x72\x73\x6d\x6e\x6f\x70\x71\x72\x73\x74\x6e\x6f\x70\x71" + "\x72\x73\x74\x75", + "\xc9\x7c\xa9\xa5\x59\x85\x0c\xe9\x7a\x04\xa9\x6d\xef\x6d\x99\xa9\xe0\xe0" + "\xe2\xab\x14\xe6\xb8\xdf\x26\x5f\xc0\xb3", + 1, 896, NULL, 0 }, + { 0, 256, 18, NULL, + "\x61\x62\x63\x64\x65\x66\x67\x68\x62\x63\x64\x65\x66\x67\x68\x69\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x64\x65\x66\x67\x68\x69\x6a\x6b\x65\x66\x67\x68" + "\x69\x6a\x6b\x6c\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70" + "\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x6c\x6d" + "\x6e\x6f\x70\x71\x72\x73\x6d\x6e\x6f\x70\x71\x72\x73\x74\x6e\x6f\x70\x71" + "\x72\x73\x74\x75", + "\xcf\x5b\x16\xa7\x78\xaf\x83\x80\x03\x6c\xe5\x9e\x7b\x04\x92\x37\x0b\x24" + "\x9b\x11\xe8\xf0\x7a\x51\xaf\xac\x45\x03\x7a\xfe\xe9\xd1", + 1, 896, NULL, 0 }, + { 0, 384, 19, NULL, + "\x61\x62\x63\x64\x65\x66\x67\x68\x62\x63\x64\x65\x66\x67\x68\x69\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x64\x65\x66\x67\x68\x69\x6a\x6b\x65\x66\x67\x68" + "\x69\x6a\x6b\x6c\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70" + "\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x6c\x6d" + "\x6e\x6f\x70\x71\x72\x73\x6d\x6e\x6f\x70\x71\x72\x73\x74\x6e\x6f\x70\x71" + "\x72\x73\x74\x75", + "\x09\x33\x0c\x33\xf7\x11\x47\xe8\x3d\x19\x2f\xc7\x82\xcd\x1b\x47\x53\x11" + "\x1b\x17\x3b\x3b\x05\xd2\x2f\xa0\x80\x86\xe3\xb0\xf7\x12\xfc\xc7\xc7\x1a" + "\x55\x7e\x2d\xb9\x66\xc3\xe9\xfa\x91\x74\x60\x39", + 1, 896, NULL, 0 }, + { 0, 512, 20, NULL, + "\x61\x62\x63\x64\x65\x66\x67\x68\x62\x63\x64\x65\x66\x67\x68\x69\x63\x64" + "\x65\x66\x67\x68\x69\x6a\x64\x65\x66\x67\x68\x69\x6a\x6b\x65\x66\x67\x68" + "\x69\x6a\x6b\x6c\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x67\x68\x69\x6a\x6b\x6c" + "\x6d\x6e\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70" + "\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x6c\x6d" + "\x6e\x6f\x70\x71\x72\x73\x6d\x6e\x6f\x70\x71\x72\x73\x74\x6e\x6f\x70\x71" + "\x72\x73\x74\x75", + "\x8e\x95\x9b\x75\xda\xe3\x13\xda\x8c\xf4\xf7\x28\x14\xfc\x14\x3f\x8f\x77" + "\x79\xc6\xeb\x9f\x7f\xa1\x72\x99\xae\xad\xb6\x88\x90\x18\x50\x1d\x28\x9e" + "\x49\x00\xf7\xe4\x33\x1b\x99\xde\xc4\xb5\x43\x3a\xc7\xd3\x29\xee\xb6\xdd" + "\x26\x54\x5e\x96\xe5\x5b\x87\x4b\xe9\x09", + 1, 896, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/snow3g_test.c b/test/kat-app/snow3g_test.c similarity index 99% rename from test/snow3g_test.c rename to test/kat-app/snow3g_test.c index a8218bf63c0c1bcd1d9de0b8fc5888b2160f1e06..fb5372fd04d1ea8527d7a096b04c7982c6ebf64d 100644 --- a/test/snow3g_test.c +++ b/test/kat-app/snow3g_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Copyright (c) 2022, Nokia Redistribution and use in source and binary forms, with or without @@ -126,7 +126,8 @@ struct { * @param ptr [IN] - pointer to beginning of buffer. * @param len [IN] - length of buffer. ******************************************************************************/ -static inline void snow3g_hexdump(const char *message, uint8_t *ptr, int len) +static inline +void snow3g_hexdump(const char *message, const uint8_t *ptr, int len) { int ctr; @@ -141,9 +142,10 @@ static inline void snow3g_hexdump(const char *message, uint8_t *ptr, int len) } static inline int -submit_uea2_jobs(struct IMB_MGR *mb_mgr, uint8_t **keys, uint8_t **ivs, - uint8_t **src, uint8_t **dst, const uint32_t *bitlens, - const uint32_t *bit_offsets, int dir, +submit_uea2_jobs(struct IMB_MGR *mb_mgr, uint8_t ** const keys, + uint8_t ** const ivs, uint8_t ** const src, + uint8_t ** const dst, const uint32_t *bitlens, + const uint32_t *bit_offsets, const int dir, const unsigned int num_jobs) { IMB_JOB *job; @@ -334,8 +336,7 @@ validate_snow3g_f8_1_block(struct IMB_MGR *mb_mgr, uint32_t job_api, pKeySched = malloc(size); if (!pKeySched) { - printf("malloc(IMB_SNOW3G_KEY_SCHED_SIZE(mb_mgr)): failed ! " - "\n"); + printf("malloc(IMB_SNOW3G_KEY_SCHED_SIZE(mb_mgr)): failed!\n"); goto snow3g_f8_1_buffer_exit; } @@ -509,8 +510,7 @@ validate_snow3g_f8_1_bitblock(struct IMB_MGR *mb_mgr, pKeySched = malloc(size); if (!pKeySched) { - printf("malloc(IMB_SNOW3G_KEY_SCHED_SIZE(mb_mgr)): failed ! " - "\n"); + printf("malloc(IMB_SNOW3G_KEY_SCHED_SIZE(mb_mgr)): failed!\n"); goto snow3g_f8_1_buffer_bit_exit; } diff --git a/test/snow3g_test_vectors.h b/test/kat-app/snow3g_test_vectors.h similarity index 99% rename from test/snow3g_test_vectors.h rename to test/kat-app/snow3g_test_vectors.h index a95a9e6b423458c0084cc11ded30cc874369dd0c..2d49299ce0621c68584341ffe203cfab0b4ebb2f 100644 --- a/test/snow3g_test_vectors.h +++ b/test/kat-app/snow3g_test_vectors.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Copyright (c) 2022, Nokia Redistribution and use in source and binary forms, with or without diff --git a/test/snow_v_test.c b/test/kat-app/snow_v_test.c similarity index 99% rename from test/snow_v_test.c rename to test/kat-app/snow_v_test.c index 35431418ce141f92bdff1c7227af6454db97ea5b..63ce5eb6d20c4a95facf3d4af1755cecf44ca8da 100644 --- a/test/snow_v_test.c +++ b/test/kat-app/snow_v_test.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2021-2022 Intel Corporation All rights reserved. + Copyright(c) 2021-2023 Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/test/test_api.py b/test/kat-app/test_api.py old mode 100755 new mode 100644 similarity index 99% rename from test/test_api.py rename to test/kat-app/test_api.py index 005e8b04f3d012face25434aca3682b9949a172d..dad2a267b7011c1529c2f6d8d50edc0615fa11a1 --- a/test/test_api.py +++ b/test/kat-app/test_api.py @@ -1,5 +1,5 @@ #***************************************************************************** -# Copyright (c) 2021-2022, Intel Corporation +# Copyright (c) 2021-2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -26,7 +26,7 @@ # ***************************************************************************** import os, re -LIB_PATH = "../lib/" +LIB_PATH = "../../lib/" LIB_NAME = "libIPSec_MB.so" LIB_HEADER = "ipsec-mb.h" COMMON_TOP_LINES = "direct_api_param_test.c_template" diff --git a/test/kat-app/win_x64.mak b/test/kat-app/win_x64.mak new file mode 100644 index 0000000000000000000000000000000000000000..0fcedb304870bb19255240d8868429935bc9659c --- /dev/null +++ b/test/kat-app/win_x64.mak @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP = imb-kat + +include ..\common\win_x64_common.mk + +TEST_OBJS = utils.obj main.obj gcm_test.obj ctr_test.obj customop_test.obj des_test.obj ccm_test.obj cmac_test.obj hmac_sha1_test.obj hmac_sha256_sha512_test.obj hmac_md5_test.obj aes_test.obj sha_test.obj chained_test.obj api_test.obj pon_test.obj ecb_test.obj zuc_test.obj kasumi_test.obj snow3g_test.obj direct_api_test.obj clear_mem_test.obj hec_test.obj xcbc_test.obj aes_cbcs_test.obj crc_test.obj chacha_test.obj poly1305_test.obj chacha20_poly1305_test.obj null_test.obj snow_v_test.obj direct_api_param_test.obj quic_ecb_test.obj hmac_sha1.json.obj hmac_sha224.json.obj hmac_sha256.json.obj hmac_sha384.json.obj hmac_sha512.json.obj hmac_md5.json.obj gmac_test.obj gmac_test.json.obj ghash_test.obj ghash_test.json.obj poly1305_test.json.obj cmac_test.json.obj xcbc_test.json.obj sha_test.json.obj +TEST_LFLAGS = /out:$(APP).exe $(DLFLAGS) + +all: $(APP).exe tests.dep + +$(APP).exe: $(TEST_OBJS) $(IPSECLIB) + $(LNK) $(TEST_LFLAGS) $(TEST_OBJS) $(IPSECLIB) + +tests.dep: $(TEST_OBJS) + @type *.obj.dep > $@ 2> nul + +.c.obj: + $(CC) /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{..\common\}.c.obj: + $(CC) /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +clean: + del /q $(TEST_OBJS) tests.dep *.obj.dep $(APP).* + +!if exist(tests.dep) +!include tests.dep +!endif diff --git a/test/kat-app/xcbc_test.c b/test/kat-app/xcbc_test.c new file mode 100644 index 0000000000000000000000000000000000000000..8761b05a51ec72eae5bfe3f5abbc71ccc0e92b65 --- /dev/null +++ b/test/kat-app/xcbc_test.c @@ -0,0 +1,281 @@ +/***************************************************************************** + Copyright (c) 2020-2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include "gcm_ctr_vectors_test.h" +#include "utils.h" +#include "mac_test.h" + +int xcbc_test(struct IMB_MGR *mb_mgr); + +extern const struct mac_test xcbc_test_json[]; + +static int +xcbc_job_ok(const struct mac_test *vec, + const struct IMB_JOB *job, + const uint8_t *auth, + const uint8_t *padding, + const size_t sizeof_padding) +{ + const size_t auth_len = job->auth_tag_output_len_in_bytes; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("%d Error status:%d", __LINE__, job->status); + return 0; + } + + /* hash checks */ + if (memcmp(padding, &auth[sizeof_padding + auth_len], + sizeof_padding)) { + printf("hash overwrite tail\n"); + hexdump(stderr, "Target", + &auth[sizeof_padding + auth_len], sizeof_padding); + return 0; + } + + if (memcmp(padding, &auth[0], sizeof_padding)) { + printf("hash overwrite head\n"); + hexdump(stderr, "Target", &auth[0], sizeof_padding); + return 0; + } + + if (memcmp((const void *) vec->tag, &auth[sizeof_padding], auth_len)) { + printf("hash mismatched\n"); + hexdump(stderr, "Received", &auth[sizeof_padding], + auth_len); + hexdump(stderr, "Expected", (const void *) vec->tag, + auth_len); + return 0; + } + return 1; +} + +static int +test_xcbc(struct IMB_MGR *mb_mgr, + const struct mac_test *vec, + const int dir, + const int num_jobs) +{ + DECLARE_ALIGNED(uint32_t k1_exp[4*11], 16); + uint8_t k2[16], k3[16]; + struct IMB_JOB *job; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + int i = 0, jobs_rx = 0, ret = -1; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + auths[i] = malloc(16 + (sizeof(padding) * 2)); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + + memset(auths[i], -1, 16 + (sizeof(padding) * 2)); + } + + IMB_AES_XCBC_KEYEXP(mb_mgr, (const void *) vec->key, k1_exp, k2, k3); + + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + /** + * Submit all jobs then flush any outstanding jobs + */ + for (i = 0; i < num_jobs; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_AES_XCBC; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.XCBC._k1_expanded = k1_exp; + job->u.XCBC._k2 = k2; + job->u.XCBC._k3 = k3; + job->src = (const void *) vec->msg; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize / 8; + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job) { + jobs_rx++; + if (num_jobs < 4) { + printf("%d Unexpected return from submit_job\n", + __LINE__); + goto end; + } + if (!xcbc_job_ok(vec, job, job->user_data, padding, + sizeof(padding))) + goto end; + } + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + jobs_rx++; + + if (!xcbc_job_ok(vec, job, job->user_data, padding, + sizeof(padding))) + goto end; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + + /** + * Submit each job and flush immediately + */ + for (i = 0; i < num_jobs; i++) { + struct IMB_JOB *first_job = NULL; + + job = IMB_GET_NEXT_JOB(mb_mgr); + first_job = job; + + job->cipher_direction = dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_AES_XCBC; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->u.XCBC._k1_expanded = k1_exp; + job->u.XCBC._k2 = k2; + job->u.XCBC._k3 = k3; + job->src = (const void *) vec->msg; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize / 8; + + job->user_data = auths[i]; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job != NULL) { + printf("Received job, expected NULL\n"); + goto end; + } + + while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { + if (job != first_job) { + printf("Invalid return job received\n"); + goto end; + } + if (!xcbc_job_ok(vec, job, job->user_data, padding, + sizeof(padding))) + goto end; + } + } + + ret = 0; + + end: + while (IMB_FLUSH_JOB(mb_mgr) != NULL) + ; + + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static void +test_xcbc_std_vectors(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx, + const int num_jobs) +{ + const struct mac_test *v = xcbc_test_json; + + if (!quiet_mode) + printf("AES-XCBC-128 standard test vectors (N jobs = %d):\n", num_jobs); + for (; v->msg != NULL; v++) { + + if (!quiet_mode) { +#ifdef DEBUG + printf("Standard XCBC-128 vector %zu Msg len: %zu, " + "Tag len:%zu\n", + v->tcId, + v->msgSize / 8, + v->tagSize / 8); +#else + printf("."); +#endif + } + + if (test_xcbc(mb_mgr, v, + IMB_DIR_ENCRYPT, num_jobs)) { + printf("error #%zu encrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_xcbc(mb_mgr, v, + IMB_DIR_DECRYPT, num_jobs)) { + printf("error #%zu decrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + } + if (!quiet_mode) + printf("\n"); +} + +int +xcbc_test(struct IMB_MGR *mb_mgr) +{ + struct test_suite_context ctx; + int i, errors; + + test_suite_start(&ctx, "AES-XCBC-128"); + /* AES-XCBC 128 with standard vectors */ + for (i = 1; i < 20; i++) + test_xcbc_std_vectors(mb_mgr, &ctx, i); + errors = test_suite_end(&ctx); + + return errors; +} diff --git a/test/kat-app/xcbc_test.json.c b/test/kat-app/xcbc_test.json.c new file mode 100644 index 0000000000000000000000000000000000000000..78a5913f242452d65fe4e62f12666910b12819f7 --- /dev/null +++ b/test/kat-app/xcbc_test.json.c @@ -0,0 +1,111 @@ +/***************************************************************************** + Copyright (c) 2023, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* XCBC, RFC3566 */ +#include "mac_test.h" + +const struct mac_test xcbc_test_json[] = { + /* Vectors from https://tools.ietf.org/html/rfc3566 */ + { 128, 96, 1, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", "", + "\x75\xf0\x25\x1d\x52\x8a\xc0\x1c\x45\x73\xdf\xd5", 1, 0, NULL, 0 }, + { 128, 96, 2, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02", "\x5b\x37\x65\x80\xae\x2f\x19\xaf\xe7\x21\x9c\xee", 1, 24, NULL, 0 }, + { 128, 96, 3, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd2\xa2\x46\xfa\x34\x9b\x68\xa7\x99\x98\xa4\x39", 1, 128, NULL, 0 }, + { 128, 96, 4, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13", + "\x47\xf5\x1b\x45\x64\x96\x62\x15\xb8\x98\x5c\x63", 1, 160, NULL, 0 }, + { 128, 96, 5, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\xf5\x4f\x0e\xc8\xd2\xb9\xf3\xd3\x68\x07\x73\x4b", 1, 256, NULL, 0 }, + { 128, 96, 6, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11" + "\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21", + "\xbe\xcb\xb3\xbc\xcd\xb5\x18\xa3\x06\x77\xd5\x48", 1, 272, NULL, 0 }, + { 128, 96, 7, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xf0\xda\xfe\xe8\x95\xdb\x30\x25\x37\x61\x10\x3b", 1, 8000, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/zuc_test.c b/test/kat-app/zuc_test.c similarity index 72% rename from test/zuc_test.c rename to test/kat-app/zuc_test.c index 219bee3c4662534e1af260383aec2c99334c75f3..259e12e73273b5355b4e13529adae6ef5869c33b 100644 --- a/test/zuc_test.c +++ b/test/kat-app/zuc_test.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -50,9 +50,12 @@ #define FAIL_STATUS -1 #define DIM(_x) (sizeof(_x)/sizeof(_x[0])) -enum test_type { - TEST_4_BUFFER, - TEST_N_BUFFER +#define MAX_BURST_JOBS 32 + +enum api_type { + TEST_DIRECT_API, + TEST_SINGLE_JOB_API, + TEST_BURST_JOB_API }; int zuc_test(struct IMB_MGR *mb_mgr); @@ -61,42 +64,34 @@ int validate_zuc_algorithm(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV); int validate_zuc_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api); + const enum api_type type); int validate_zuc_EEA_4_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, - uint8_t **pIV, const unsigned int job_api); + uint8_t **pIV, enum api_type type); int validate_zuc_EEA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs, const unsigned int job_api); + uint32_t numBuffs, const enum api_type type); int validate_zuc_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api); + const enum api_type type); int validate_zuc_EIA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, uint32_t numBuffs, - const unsigned int job_api); + const enum api_type type); #ifdef __aarch64__ int validate_zuc256_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api); + const enum api_type type); int validate_zuc256_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api); -int validate_zuc256_EEA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, - uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs, const unsigned int job_api); -int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, - uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs, const unsigned int job_api); -#else + const enum api_type type); +#endif int validate_zuc256_EEA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs); + uint32_t numBuffs, const enum api_type type); int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs); - -#endif + uint32_t numBuffs, const enum api_type type); static void byte_hexdump(const char *message, const uint8_t *ptr, int len); @@ -118,7 +113,7 @@ static uint32_t createData(uint8_t *pSrcData[MAXBUFS], pSrcData[i] = (uint8_t *)malloc(MAX_BUFFER_LENGTH_IN_BYTES); if (!pSrcData[i]) { - uint32_t j = 0; + uint32_t j; printf("malloc(pSrcData[i]): failed!\n"); @@ -152,7 +147,7 @@ static uint32_t createKeyVecData(uint32_t keyLen, uint8_t *pKeys[MAXBUFS], uint32_t i = 0; for (i = 0; i < numOfBuffs; i++) { - uint32_t j = 0; + uint32_t j; pIV[i] = (uint8_t *)malloc(ivLen); @@ -247,8 +242,8 @@ int zuc_test(struct IMB_MGR *mb_mgr) } /* Create random keys and vectors */ - if (createKeyVecData(ZUC256_KEY_LEN_IN_BYTES, pKeys, - ZUC256_IV_LEN_IN_BYTES, + if (createKeyVecData(IMB_ZUC256_KEY_LEN_IN_BYTES, pKeys, + IMB_ZUC256_IV_LEN_IN_BYTES_MAX, pIV, MAXBUFS)) { printf("createKeyVecData() error\n"); freePtrArray(pSrcData, MAXBUFS); @@ -266,64 +261,63 @@ int zuc_test(struct IMB_MGR *mb_mgr) /* Direct API tests */ if (validate_zuc_EEA_1_block(mb_mgr, pSrcData[0], pSrcData[0], pKeys[0], - pIV[0], 0)) + pIV[0], TEST_DIRECT_API)) test_suite_update(&eea3_ctx, 0, 1); else test_suite_update(&eea3_ctx, 1, 0); - if (validate_zuc_EEA_4_block(mb_mgr, pSrcData, pSrcData, pKeys, pIV, 0)) + if (validate_zuc_EEA_4_block(mb_mgr, pSrcData, pSrcData, pKeys, pIV, + TEST_DIRECT_API)) test_suite_update(&eea3_ctx, 0, 1); else test_suite_update(&eea3_ctx, 1, 0); for (i = 0; i < DIM(numBuffs); i++) { if (validate_zuc_EEA_n_block(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 0)) + pIV, numBuffs[i], TEST_DIRECT_API)) test_suite_update(&eea3_ctx, 0, 1); else test_suite_update(&eea3_ctx, 1, 0); } if (validate_zuc_EIA_1_block(mb_mgr, pSrcData[0], pDstData[0], pKeys[0], - pIV[0], 0)) + pIV[0], TEST_DIRECT_API)) test_suite_update(&eia3_ctx, 0, 1); else test_suite_update(&eia3_ctx, 1, 0); for (i = 0; i < DIM(numBuffs); i++) { if (validate_zuc_EIA_n_block(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 0)) + pIV, numBuffs[i], TEST_DIRECT_API)) test_suite_update(&eia3_ctx, 0, 1); else test_suite_update(&eia3_ctx, 1, 0); } #ifdef __aarch64__ - /* ZUC-EEA3-256 tests */ if (validate_zuc256_EEA_1_block(mb_mgr, pSrcData[0], pDstData[0], pKeys[0], - pIV[0], 0)) + pIV[0], TEST_DIRECT_API)) test_suite_update(&eea3_256_ctx, 0, 1); else test_suite_update(&eea3_256_ctx, 1, 0); for (i = 0; i < DIM(numBuffs); i++) { if (validate_zuc256_EEA3(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 0)) + pIV, numBuffs[i], TEST_DIRECT_API)) test_suite_update(&eea3_256_ctx, 0, 1); else test_suite_update(&eea3_256_ctx, 1, 0); } - /* ZUC-EIA3-256 tests */ if (validate_zuc256_EIA_1_block(mb_mgr, pSrcData[0], pDstData[0], pKeys[0], - pIV[0], 0)) + pIV[0], TEST_DIRECT_API)) test_suite_update(&eia3_256_ctx, 0, 1); else test_suite_update(&eia3_256_ctx, 1, 0); for (i = 0; i < DIM(numBuffs); i++) { if (validate_zuc256_EIA3(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 0)) + pIV, numBuffs[i], TEST_DIRECT_API)) test_suite_update(&eia3_256_ctx, 0, 1); else test_suite_update(&eia3_256_ctx, 1, 0); @@ -332,82 +326,95 @@ int zuc_test(struct IMB_MGR *mb_mgr) /* Job API tests */ if (validate_zuc_EEA_1_block(mb_mgr, pSrcData[0], pSrcData[0], pKeys[0], - pIV[0], 1)) + pIV[0], TEST_SINGLE_JOB_API)) test_suite_update(&eea3_ctx, 0, 1); else test_suite_update(&eea3_ctx, 1, 0); - if (validate_zuc_EEA_4_block(mb_mgr, pSrcData, pSrcData, pKeys, pIV, 1)) + if (validate_zuc_EEA_4_block(mb_mgr, pSrcData, pSrcData, pKeys, pIV, + TEST_SINGLE_JOB_API)) test_suite_update(&eea3_ctx, 0, 1); else test_suite_update(&eea3_ctx, 1, 0); for (i = 0; i < DIM(numBuffs); i++) { if (validate_zuc_EEA_n_block(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 1)) + pIV, numBuffs[i], TEST_SINGLE_JOB_API)) test_suite_update(&eea3_ctx, 0, 1); else test_suite_update(&eea3_ctx, 1, 0); } + for (i = 0; i < DIM(numBuffs); i++) { + if (validate_zuc256_EEA3(mb_mgr, pSrcData, pDstData, pKeys, + pIV, numBuffs[i], TEST_SINGLE_JOB_API)) + test_suite_update(&eea3_256_ctx, 0, 1); + else + test_suite_update(&eea3_256_ctx, 1, 0); + } + if (validate_zuc_EIA_1_block(mb_mgr, pSrcData[0], pDstData[0], pKeys[0], - pIV[0], 1)) + pIV[0], TEST_SINGLE_JOB_API)) test_suite_update(&eia3_ctx, 0, 1); else test_suite_update(&eia3_ctx, 1, 0); for (i = 0; i < DIM(numBuffs); i++) { if (validate_zuc_EIA_n_block(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 1)) + pIV, numBuffs[i], TEST_SINGLE_JOB_API)) test_suite_update(&eia3_ctx, 0, 1); else test_suite_update(&eia3_ctx, 1, 0); } + for (i = 0; i < DIM(numBuffs); i++) { + if (validate_zuc256_EIA3(mb_mgr, pSrcData, pDstData, pKeys, + pIV, numBuffs[i], TEST_SINGLE_JOB_API)) + test_suite_update(&eia3_256_ctx, 0, 1); + else + test_suite_update(&eia3_256_ctx, 1, 0); + } + #ifdef __aarch64__ - /* ZUC-EEA3-256 tests */ if (validate_zuc256_EEA_1_block(mb_mgr, pSrcData[0], pDstData[0], pKeys[0], - pIV[0], 1)) + pIV[0], TEST_SINGLE_JOB_API)) test_suite_update(&eea3_256_ctx, 0, 1); else test_suite_update(&eea3_256_ctx, 1, 0); - /* ZUC-EIA3-256 tests */ if (validate_zuc256_EIA_1_block(mb_mgr, pSrcData[0], pDstData[0], pKeys[0], - pIV[0], 1)) + pIV[0], TEST_SINGLE_JOB_API)) test_suite_update(&eia3_256_ctx, 0, 1); else test_suite_update(&eia3_256_ctx, 1, 0); #endif - /* ZUC-EEA3-256 tests */ + /* Burst job API tests */ + for (i = 0; i < DIM(numBuffs); i++) { + if (validate_zuc_EEA_n_block(mb_mgr, pSrcData, pDstData, pKeys, + pIV, numBuffs[i], TEST_BURST_JOB_API)) + test_suite_update(&eea3_ctx, 0, 1); + else + test_suite_update(&eea3_ctx, 1, 0); + } + for (i = 0; i < DIM(numBuffs); i++) { -#ifdef __aarch64__ - if (validate_zuc256_EEA3(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 1)) -#else if (validate_zuc256_EEA3(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i])) -#endif + pIV, numBuffs[i], TEST_BURST_JOB_API)) test_suite_update(&eea3_256_ctx, 0, 1); else test_suite_update(&eea3_256_ctx, 1, 0); } - /* ZUC-EIA3-256 tests */ for (i = 0; i < DIM(numBuffs); i++) { -#ifdef __aarch64__ - if (validate_zuc256_EIA3(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i], 1)) -#else if (validate_zuc256_EIA3(mb_mgr, pSrcData, pDstData, pKeys, - pIV, numBuffs[i])) -#endif + pIV, numBuffs[i], TEST_BURST_JOB_API)) test_suite_update(&eia3_256_ctx, 0, 1); else test_suite_update(&eia3_256_ctx, 1, 0); } + exit_zuc_test: freePtrArray(pKeys, MAXBUFS); /*Free the key buffers*/ freePtrArray(pIV, MAXBUFS); /*Free the vector buffers*/ @@ -423,8 +430,81 @@ exit_zuc_test: } static inline int -submit_eea3_jobs(struct IMB_MGR *mb_mgr, uint8_t **keys, uint8_t **ivs, - uint8_t **src, uint8_t **dst, const uint32_t *lens, +submit_burst_eea3_jobs(struct IMB_MGR *mb_mgr, uint8_t ** const keys, + uint8_t ** const ivs, uint8_t ** const src, + uint8_t ** const dst, const uint32_t *lens, + int dir, const unsigned int num_jobs, + const unsigned int key_len, + const unsigned int *iv_lens) +{ + IMB_JOB *job, *jobs[MAX_BURST_JOBS] = {NULL}; + unsigned int i; + unsigned int jobs_rx = 0; + uint32_t completed_jobs = 0; + int err; + + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + job->cipher_direction = dir; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->cipher_mode = IMB_CIPHER_ZUC_EEA3; + job->src = src[i]; + job->dst = dst[i]; + job->iv = ivs[i]; + job->iv_len_in_bytes = iv_lens[i]; + job->enc_keys = keys[i]; + job->key_len_in_bytes = key_len; + + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = lens[i]; + job->hash_alg = IMB_AUTH_NULL; + + imb_set_session(mb_mgr, job); + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + return -1; + } + +check_eea3_burst_jobs: + for (i = 0; i < completed_jobs; i++) { + job = jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i+1); + return -1; + } + + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, + num_jobs - completed_jobs, + jobs); + if (completed_jobs == 0) { + printf("Expected %u jobs, received %u\n", + num_jobs, jobs_rx); + return -1; + } + goto check_eea3_burst_jobs; + } + + return 0; +} + +static inline int +submit_eea3_jobs(struct IMB_MGR *mb_mgr, uint8_t ** const keys, + uint8_t ** const ivs, uint8_t ** const src, + uint8_t ** const dst, const uint32_t *lens, int dir, const unsigned int num_jobs, const unsigned int key_len, const unsigned int *iv_lens) @@ -478,8 +558,88 @@ submit_eea3_jobs(struct IMB_MGR *mb_mgr, uint8_t **keys, uint8_t **ivs, } static inline int -submit_eia3_jobs(struct IMB_MGR *mb_mgr, uint8_t **keys, uint8_t **iv, - uint8_t **src, uint8_t **tags, const uint32_t *lens, +submit_burst_eia3_jobs(struct IMB_MGR *mb_mgr, uint8_t ** const keys, + uint8_t ** const iv, uint8_t ** const src, + uint8_t ** const tags, const uint32_t *lens, + const unsigned int num_jobs, + const unsigned int key_sz, + const unsigned int tag_sz, + const unsigned int *iv_lens) +{ + IMB_JOB *job, *jobs[MAX_BURST_JOBS] = {NULL}; + unsigned int i; + unsigned int jobs_rx = 0; + uint32_t completed_jobs = 0; + int err; + + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->cipher_mode = IMB_CIPHER_NULL; + job->src = src[i]; + if (iv_lens[i] == IMB_ZUC256_IV_LEN_IN_BYTES_MIN) { + job->u.ZUC_EIA3._iv = NULL; + job->u.ZUC_EIA3._iv23 = iv[i]; + } else { + job->u.ZUC_EIA3._iv = iv[i]; + job->u.ZUC_EIA3._iv23 = NULL; + } + job->u.ZUC_EIA3._key = keys[i]; + + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bits = lens[i]; + if (key_sz == IMB_ZUC_KEY_LEN_IN_BYTES) + job->hash_alg = IMB_AUTH_ZUC_EIA3_BITLEN; + else + job->hash_alg = IMB_AUTH_ZUC256_EIA3_BITLEN; + job->auth_tag_output = tags[i]; + job->auth_tag_output_len_in_bytes = tag_sz; + + imb_set_session(mb_mgr, job); + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + return -1; + } + +check_eia3_burst_jobs: + for (i = 0; i < completed_jobs; i++) { + job = jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i+1); + return -1; + } + + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, + num_jobs - completed_jobs, + jobs); + if (completed_jobs == 0) { + printf("Expected %u jobs, received %u\n", + num_jobs, jobs_rx); + return -1; + } + goto check_eia3_burst_jobs; + } + return 0; +} + +static inline int +submit_eia3_jobs(struct IMB_MGR *mb_mgr, uint8_t ** const keys, + uint8_t ** const iv, uint8_t ** const src, + uint8_t ** const tags, const uint32_t *lens, const unsigned int num_jobs, const unsigned int key_sz, const unsigned int tag_sz, @@ -494,7 +654,7 @@ submit_eia3_jobs(struct IMB_MGR *mb_mgr, uint8_t **keys, uint8_t **iv, job->chain_order = IMB_ORDER_CIPHER_HASH; job->cipher_mode = IMB_CIPHER_NULL; job->src = src[i]; - if (iv_lens[i] == 23) { + if (iv_lens[i] == IMB_ZUC256_IV_LEN_IN_BYTES_MIN) { job->u.ZUC_EIA3._iv = NULL; job->u.ZUC_EIA3._iv23 = iv[i]; } else { @@ -505,7 +665,7 @@ submit_eia3_jobs(struct IMB_MGR *mb_mgr, uint8_t **keys, uint8_t **iv, job->hash_start_src_offset_in_bytes = 0; job->msg_len_to_hash_in_bits = lens[i]; - if (key_sz == 16) + if (key_sz == IMB_ZUC_KEY_LEN_IN_BYTES) job->hash_alg = IMB_AUTH_ZUC_EIA3_BITLEN; else job->hash_alg = IMB_AUTH_ZUC256_EIA3_BITLEN; @@ -576,13 +736,17 @@ test_output(const uint8_t *out, const uint8_t *ref, const uint32_t bytelen, ret = -1; } #ifdef DEBUG - else - printf("%s : PASS\n", err_msg); + else { + if (!quiet_mode) + printf("%s : PASS\n", err_msg); + } #endif } #ifdef DEBUG - else - printf("%s : PASS\n", err_msg); + else { + if (!quiet_mode) + printf("%s : PASS\n", err_msg); + } #endif fflush(stdout); @@ -592,7 +756,7 @@ test_output(const uint8_t *out, const uint8_t *ref, const uint32_t bytelen, int validate_zuc_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api) + const enum api_type type) { uint32_t i; int ret = 0; @@ -602,20 +766,20 @@ validate_zuc_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, char msg[50]; int retTmp; uint32_t byteLength; - const unsigned int iv_len = ZUC_IV_LEN_IN_BYTES; + const unsigned int iv_len = IMB_ZUC_IV_LEN_IN_BYTES; - memcpy(pKeys, testEEA3_vectors[i].CK, ZUC_KEY_LEN_IN_BYTES); + memcpy(pKeys, testEEA3_vectors[i].CK, IMB_ZUC_KEY_LEN_IN_BYTES); zuc_eea3_iv_gen(testEEA3_vectors[i].count, testEEA3_vectors[i].Bearer, testEEA3_vectors[i].Direction, pIV); byteLength = (testEEA3_vectors[i].length_in_bits + 7) / 8; memcpy(pSrcData, testEEA3_vectors[i].plaintext, byteLength); - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eea3_jobs(mb_mgr, &pKeys, &pIV, &pSrcData, &pDstData, &byteLength, IMB_DIR_ENCRYPT, 1, - ZUC_KEY_LEN_IN_BYTES, + IMB_ZUC_KEY_LEN_IN_BYTES, &iv_len); else IMB_ZUC_EEA3_1_BUFFER(mb_mgr, pKeys, pIV, pSrcData, @@ -636,8 +800,8 @@ validate_zuc_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, static int submit_and_verify(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - const unsigned int job_api, IMB_CIPHER_DIRECTION dir, - enum test_type type, const unsigned int var_bufs, + const enum api_type type, IMB_CIPHER_DIRECTION dir, + const unsigned int var_bufs, const unsigned int num_buffers, const uint32_t *buf_idx) { unsigned int i; @@ -649,28 +813,32 @@ submit_and_verify(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (i = 0; i < num_buffers; i++) { vector = &testEEA3_vectors[buf_idx[i]]; packetLen[i] = (vector->length_in_bits + 7) / 8; - memcpy(pKeys[i], vector->CK, ZUC_KEY_LEN_IN_BYTES); + memcpy(pKeys[i], vector->CK, IMB_ZUC_KEY_LEN_IN_BYTES); zuc_eea3_iv_gen(vector->count, vector->Bearer, vector->Direction, pIV[i]); - iv_lens[i] = ZUC_IV_LEN_IN_BYTES; + iv_lens[i] = IMB_ZUC_IV_LEN_IN_BYTES; if (dir == IMB_DIR_ENCRYPT) memcpy(pSrcData[i], vector->plaintext, packetLen[i]); else memcpy(pSrcData[i], vector->ciphertext, packetLen[i]); } - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eea3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, packetLen, dir, num_buffers, - ZUC_KEY_LEN_IN_BYTES, iv_lens); + IMB_ZUC_KEY_LEN_IN_BYTES, iv_lens); + else if (type == TEST_BURST_JOB_API) + submit_burst_eea3_jobs(mb_mgr, pKeys, pIV, pSrcData, + pDstData, packetLen, dir, num_buffers, + IMB_ZUC_KEY_LEN_IN_BYTES, iv_lens); else { - if (type == TEST_4_BUFFER) + if (num_buffers == 4) IMB_ZUC_EEA3_4_BUFFER(mb_mgr, (const void * const *)pKeys, (const void * const *)pIV, (const void * const *)pSrcData, (void **)pDstData, packetLen); - else /* TEST_N_BUFFER */ + else IMB_ZUC_EEA3_N_BUFFER(mb_mgr, (const void * const *)pKeys, (const void * const *)pIV, @@ -689,11 +857,11 @@ submit_and_verify(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, if (var_bufs) snprintf(msg_start, sizeof(msg_start), "Validate ZUC %c block multi-vector", - type == TEST_4_BUFFER ? '4' : 'N'); + num_buffers == 4 ? '4' : 'N'); else snprintf(msg_start, sizeof(msg_start), "Validate ZUC %c block", - type == TEST_4_BUFFER ? '4' : 'N'); + num_buffers == 4 ? '4' : 'N'); if (dir == IMB_DIR_ENCRYPT) { snprintf(msg, sizeof(msg), @@ -717,24 +885,13 @@ submit_and_verify(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, return ret; } -#ifdef __aarch64__ static int submit_and_verify_zuc256(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - IMB_CIPHER_DIRECTION dir, - const unsigned int var_bufs, - const unsigned int num_buffers, - const uint32_t *buf_idx, - const unsigned int job_api) -#else -static int -submit_and_verify_zuc256(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, - uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - IMB_CIPHER_DIRECTION dir, + const enum api_type type, IMB_CIPHER_DIRECTION dir, const unsigned int var_bufs, const unsigned int num_buffers, const uint32_t *buf_idx) -#endif { unsigned int i; uint32_t packetLen[MAXBUFS]; @@ -745,7 +902,7 @@ submit_and_verify_zuc256(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (i = 0; i < num_buffers; i++) { vector = &test256EEA3_vectors[buf_idx[i]]; packetLen[i] = (vector->length_in_bits + 7) / 8; - memcpy(pKeys[i], vector->CK, ZUC256_KEY_LEN_IN_BYTES); + memcpy(pKeys[i], vector->CK, IMB_ZUC256_KEY_LEN_IN_BYTES); memcpy(pIV[i], vector->IV, vector->iv_length); if (dir == IMB_DIR_ENCRYPT) memcpy(pSrcData[i], vector->plaintext, packetLen[i]); @@ -754,12 +911,16 @@ submit_and_verify_zuc256(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, iv_lens[i] = vector->iv_length; } -#ifdef __aarch64__ - if (job_api) { + if (type == TEST_SINGLE_JOB_API) submit_eea3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, packetLen, dir, num_buffers, - ZUC256_KEY_LEN_IN_BYTES, iv_lens); - } else { + IMB_ZUC256_KEY_LEN_IN_BYTES, iv_lens); + else if (type == TEST_BURST_JOB_API) + submit_burst_eea3_jobs(mb_mgr, pKeys, pIV, pSrcData, + pDstData, packetLen, dir, num_buffers, + IMB_ZUC256_KEY_LEN_IN_BYTES, iv_lens); +#ifdef __aarch64__ + else IMB_ZUC256_EEA3_N_BUFFER(mb_mgr, (const void * const *)pKeys, (const void * const *)pIV, @@ -768,11 +929,6 @@ submit_and_verify_zuc256(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, (void **)pDstData, packetLen, num_buffers); - } -#else - submit_eea3_jobs(mb_mgr, pKeys, pIV, pSrcData, - pDstData, packetLen, dir, num_buffers, - ZUC256_KEY_LEN_IN_BYTES, iv_lens); #endif for (i = 0; i < num_buffers; i++) { @@ -813,7 +969,7 @@ submit_and_verify_zuc256(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, int validate_zuc_EEA_4_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - const unsigned int job_api) + const enum api_type type) { uint32_t i, j; int ret = 0; @@ -825,13 +981,13 @@ int validate_zuc_EEA_4_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, buf_idx[j] = i; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, IMB_DIR_ENCRYPT, - TEST_4_BUFFER, 0, 4, buf_idx); + pIV, type, IMB_DIR_ENCRYPT, + 0, 4, buf_idx); if (retTmp < 0) ret = retTmp; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, IMB_DIR_DECRYPT, - TEST_4_BUFFER, 0, 4, buf_idx); + pIV, type, IMB_DIR_DECRYPT, + 0, 4, buf_idx); if (retTmp < 0) ret = retTmp; } @@ -843,13 +999,13 @@ int validate_zuc_EEA_4_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, buf_idx[j] = i+j; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, IMB_DIR_ENCRYPT, - TEST_4_BUFFER, 1, 4, buf_idx); + pIV, type, IMB_DIR_ENCRYPT, + 1, 4, buf_idx); if (retTmp < 0) ret = retTmp; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, IMB_DIR_DECRYPT, - TEST_4_BUFFER, 1, 4, buf_idx); + pIV, type, IMB_DIR_DECRYPT, + 1, 4, buf_idx); if (retTmp < 0) ret = retTmp; } @@ -859,7 +1015,7 @@ int validate_zuc_EEA_4_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, int validate_zuc_EEA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs, const unsigned int job_api) + uint32_t numBuffs, const enum api_type type) { uint32_t i, j; int ret = 0; @@ -872,15 +1028,15 @@ int validate_zuc_EEA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, buf_idx[j] = i; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, - IMB_DIR_ENCRYPT, TEST_N_BUFFER, + pIV, type, + IMB_DIR_ENCRYPT, 0, numBuffs, buf_idx); if (retTmp < 0) ret = retTmp; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, IMB_DIR_DECRYPT, - TEST_N_BUFFER, 0, numBuffs, buf_idx); + pIV, type, IMB_DIR_DECRYPT, + 0, numBuffs, buf_idx); if (retTmp < 0) ret = retTmp; } @@ -890,13 +1046,13 @@ int validate_zuc_EEA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, buf_idx[i] = i % NUM_ZUC_EEA3_TESTS; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, IMB_DIR_ENCRYPT, TEST_N_BUFFER, + pIV, type, IMB_DIR_ENCRYPT, 1, numBuffs, buf_idx); if (retTmp < 0) ret = retTmp; retTmp = submit_and_verify(mb_mgr, pSrcData, pDstData, pKeys, - pIV, job_api, IMB_DIR_DECRYPT, TEST_N_BUFFER, + pIV, type, IMB_DIR_DECRYPT, 1, numBuffs, buf_idx); if (retTmp < 0) ret = retTmp; @@ -908,7 +1064,7 @@ int validate_zuc_EEA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, int validate_zuc256_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api) + const enum api_type type) { uint32_t i; int ret = 0; @@ -921,17 +1077,17 @@ validate_zuc256_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint32_t iv_len; byteLength = (test256EEA3_vectors[i].length_in_bits + 7) / 8; iv_len = test256EEA3_vectors[i].iv_length; - memcpy(pKeys, test256EEA3_vectors[i].CK, ZUC256_KEY_LEN_IN_BYTES); + memcpy(pKeys, test256EEA3_vectors[i].CK, IMB_ZUC256_KEY_LEN_IN_BYTES); memcpy(pIV, test256EEA3_vectors[i].IV, iv_len); memcpy(pSrcData, test256EEA3_vectors[i].plaintext, byteLength); - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eea3_jobs(mb_mgr, &pKeys, &pIV, &pSrcData, &pDstData, &byteLength, IMB_DIR_ENCRYPT, 1, - ZUC256_KEY_LEN_IN_BYTES, + IMB_ZUC256_KEY_LEN_IN_BYTES, &iv_len); - else + else if (type == TEST_DIRECT_API) IMB_ZUC256_EEA3_1_BUFFER(mb_mgr, pKeys, pIV, iv_len, pSrcData, pDstData, byteLength); @@ -948,15 +1104,9 @@ validate_zuc256_EEA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, }; #endif -#ifdef __aarch64__ int validate_zuc256_EEA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs, const unsigned int job_api) -#else -int validate_zuc256_EEA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, - uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs) -#endif + uint32_t numBuffs, const enum api_type type) { uint32_t i, j; int ret = 0; @@ -968,28 +1118,15 @@ int validate_zuc256_EEA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (j = 0; j < numBuffs; j++) buf_idx[j] = i; - -#ifdef __aarch64__ - retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, - pKeys, pIV, IMB_DIR_ENCRYPT, - 0, numBuffs, buf_idx, job_api); -#else retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, - pKeys, pIV, IMB_DIR_ENCRYPT, + pKeys, pIV, type, IMB_DIR_ENCRYPT, 0, numBuffs, buf_idx); -#endif if (retTmp < 0) ret = retTmp; -#ifdef __aarch64__ - retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, - pKeys, pIV, IMB_DIR_DECRYPT, - 0, numBuffs, buf_idx, job_api); -#else retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, - pKeys, pIV, IMB_DIR_DECRYPT, + pKeys, pIV, type, IMB_DIR_DECRYPT, 0, numBuffs, buf_idx); -#endif if (retTmp < 0) ret = retTmp; } @@ -998,27 +1135,15 @@ int validate_zuc256_EEA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (i = 0; i < numBuffs; i++) buf_idx[i] = i % NUM_ZUC_256_EEA3_TESTS; -#ifdef __aarch64__ retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, pKeys, - pIV, IMB_DIR_ENCRYPT, - 1, numBuffs, buf_idx, job_api); -#else - retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, pKeys, - pIV, IMB_DIR_ENCRYPT, + pIV, type, IMB_DIR_ENCRYPT, 1, numBuffs, buf_idx); -#endif if (retTmp < 0) ret = retTmp; -#ifdef __aarch64__ retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, pKeys, - pIV, IMB_DIR_DECRYPT, - 1, numBuffs, buf_idx, job_api); -#else - retTmp = submit_and_verify_zuc256(mb_mgr, pSrcData, pDstData, pKeys, - pIV, IMB_DIR_DECRYPT, + pIV, type, IMB_DIR_DECRYPT, 1, numBuffs, buf_idx); -#endif if (retTmp < 0) ret = retTmp; @@ -1027,16 +1152,16 @@ int validate_zuc256_EEA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, int validate_zuc_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api) + const enum api_type type) { uint32_t i; int ret = 0; uint32_t bitLength; for (i = 0; i < NUM_ZUC_EIA3_TESTS; i++) { - const unsigned int iv_len = ZUC_IV_LEN_IN_BYTES; + const unsigned int iv_len = IMB_ZUC_IV_LEN_IN_BYTES; - memcpy(pKeys, testEIA3_vectors[i].CK, ZUC_KEY_LEN_IN_BYTES); + memcpy(pKeys, testEIA3_vectors[i].CK, IMB_ZUC_KEY_LEN_IN_BYTES); zuc_eia3_iv_gen(testEIA3_vectors[i].count, testEIA3_vectors[i].Bearer, @@ -1047,12 +1172,12 @@ int validate_zuc_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, const uint32_t byteLength = (bitLength + 7) / 8; memcpy(pSrcData, testEIA3_vectors[i].message, byteLength); - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eia3_jobs(mb_mgr, &pKeys, &pIV, &pSrcData, &pDstData, - &bitLength, 1, ZUC_KEY_LEN_IN_BYTES, - ZUC_DIGEST_LEN, &iv_len); - else + &bitLength, 1, IMB_ZUC_KEY_LEN_IN_BYTES, + IMB_ZUC_DIGEST_LEN_IN_BYTES, &iv_len); + else /* TEST_DIRECT_API */ IMB_ZUC_EIA3_1_BUFFER(mb_mgr, pKeys, pIV, pSrcData, bitLength, (uint32_t *)pDstData); const int retTmp = @@ -1063,14 +1188,17 @@ int validate_zuc_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, i + 1); byte_hexdump("Expected", (const uint8_t *)&testEIA3_vectors[i].mac, - ZUC_DIGEST_LEN); - byte_hexdump("Found", pDstData, ZUC_DIGEST_LEN); + IMB_ZUC_DIGEST_LEN_IN_BYTES); + byte_hexdump("Found", pDstData, IMB_ZUC_DIGEST_LEN_IN_BYTES); ret = retTmp; } #ifdef DEBUG - else - printf("Validate ZUC 1 block test %u (Int): PASS\n", - i + 1); + else { + if (!quiet_mode) + printf("Validate ZUC 1 block test %u (Int): " + "PASS\n", + i + 1); + } #endif fflush(stdout); } @@ -1079,7 +1207,7 @@ int validate_zuc_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, int validate_zuc_EIA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs, const unsigned int job_api) + uint32_t numBuffs, const enum api_type type) { uint32_t i, j; int retTmp, ret = 0; @@ -1091,22 +1219,30 @@ int validate_zuc_EIA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (i = 0; i < NUM_ZUC_EIA3_TESTS; i++) { vector = testEIA3_vectors[i]; for (j = 0; j < numBuffs; j++) { - memcpy(pKeys[j], vector.CK, ZUC_KEY_LEN_IN_BYTES); + memcpy(pKeys[j], vector.CK, IMB_ZUC_KEY_LEN_IN_BYTES); zuc_eia3_iv_gen(vector.count, vector.Bearer, vector.Direction, pIV[j]); bitLength[j] = vector.length_in_bits; byteLength = (bitLength[j] + 7) / 8; memcpy(pSrcData[j], vector.message, byteLength); - iv_lens[j] = ZUC_IV_LEN_IN_BYTES; + iv_lens[j] = IMB_ZUC_IV_LEN_IN_BYTES; } - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eia3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, bitLength, numBuffs, - ZUC_KEY_LEN_IN_BYTES, ZUC_DIGEST_LEN, + IMB_ZUC_KEY_LEN_IN_BYTES, + IMB_ZUC_DIGEST_LEN_IN_BYTES, iv_lens); - else + else if (type == TEST_BURST_JOB_API) + submit_burst_eia3_jobs(mb_mgr, pKeys, pIV, + pSrcData, pDstData, + bitLength, numBuffs, + IMB_ZUC_KEY_LEN_IN_BYTES, + IMB_ZUC_DIGEST_LEN_IN_BYTES, + iv_lens); + else /* TEST_BURST_JOB_API */ IMB_ZUC_EIA3_N_BUFFER(mb_mgr, (const void * const *)pKeys, (const void * const *)pIV, @@ -1116,21 +1252,24 @@ int validate_zuc_EIA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (j = 0; j < numBuffs; j++) { retTmp = - memcmp(pDstData[j], &vector.mac, ZUC_DIGEST_LEN); + memcmp(pDstData[j], &vector.mac, IMB_ZUC_DIGEST_LEN_IN_BYTES); if (retTmp) { printf("Validate ZUC n block test %u, index %u " "(Int): FAIL\n", i + 1, j); byte_hexdump("Expected", (const uint8_t *)&vector.mac, - ZUC_DIGEST_LEN); + IMB_ZUC_DIGEST_LEN_IN_BYTES); byte_hexdump("Found", pDstData[j], - ZUC_DIGEST_LEN); + IMB_ZUC_DIGEST_LEN_IN_BYTES); ret = retTmp; } #ifdef DEBUG - else - printf("Validate ZUC n block test %u, index %u " - "(Int): PASS\n", i + 1, j); + else { + if (!quiet_mode) + printf("Validate ZUC n block test %u, " + "index %u (Int): PASS\n", + i + 1, j); + } #endif fflush(stdout); } @@ -1141,23 +1280,31 @@ int validate_zuc_EIA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (i = 0; i < numBuffs; i++) { vector = testEIA3_vectors[i % NUM_ZUC_EIA3_TESTS]; memcpy(pKeys[i], vector.CK, - ZUC_KEY_LEN_IN_BYTES); + IMB_ZUC_KEY_LEN_IN_BYTES); zuc_eia3_iv_gen(vector.count, vector.Bearer, vector.Direction, pIV[i]); bitLength[i] = vector.length_in_bits; byteLength = (bitLength[i] + 7) / 8; memcpy(pSrcData[i], vector.message, byteLength); - iv_lens[j] = ZUC_IV_LEN_IN_BYTES; + iv_lens[j] = IMB_ZUC_IV_LEN_IN_BYTES; } - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eia3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, bitLength, numBuffs, - ZUC_KEY_LEN_IN_BYTES, ZUC_DIGEST_LEN, + IMB_ZUC_KEY_LEN_IN_BYTES, + IMB_ZUC_DIGEST_LEN_IN_BYTES, iv_lens); - else + else if (type == TEST_BURST_JOB_API) + submit_burst_eia3_jobs(mb_mgr, pKeys, pIV, + pSrcData, pDstData, + bitLength, numBuffs, + IMB_ZUC_KEY_LEN_IN_BYTES, + IMB_ZUC_DIGEST_LEN_IN_BYTES, + iv_lens); + else /* TEST_BURST_JOB_API */ IMB_ZUC_EIA3_N_BUFFER(mb_mgr, (const void * const *)pKeys, (const void * const *)pIV, @@ -1176,15 +1323,17 @@ int validate_zuc_EIA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, numBuffs, i); byte_hexdump("Expected", (const uint8_t *)&vector.mac, - ZUC_DIGEST_LEN); - byte_hexdump("Found", pDstData[i], ZUC_DIGEST_LEN); + IMB_ZUC_DIGEST_LEN_IN_BYTES); + byte_hexdump("Found", pDstData[i], IMB_ZUC_DIGEST_LEN_IN_BYTES); ret = retTmp; } #ifdef DEBUG - else - printf("Validate ZUC n block multi-vector test, " - "# jobs = %u, index %u (Int): PASS\n", - numBuffs, i); + else { + if (!quiet_mode) + printf("Validate ZUC n block multi-vector test," + " #jobs = %u, index %u (Int): PASS\n", + numBuffs, i); + } #endif fflush(stdout); } @@ -1225,15 +1374,18 @@ verify_tag_256(void *mac, const struct test256EIA3_vectors_t *vector, } #ifdef DEBUG else { - if (multi_vector) { - printf("Validate ZUC-256 n block multi-vector test " - "# jobs = %u, index %u (Int - %u bytes): PASS\n", - test_idx, vector_idx, tag_sz); + if (!quiet_mode) { + if (multi_vector) { + printf("Validate ZUC-256 n block multi-vector " + "test #jobs = %u, index %u " + "(Int - %u bytes): PASS\n", + test_idx, vector_idx, tag_sz); - } else { - printf("Validate ZUC-256 n block test %u, " - "index %u (Int - %u bytes): PASS\n", - test_idx + 1, vector_idx, tag_sz); + } else { + printf("Validate ZUC-256 n block test %u, " + "index %u (Int - %u bytes): PASS\n", + test_idx + 1, vector_idx, tag_sz); + } } } #endif @@ -1245,7 +1397,7 @@ verify_tag_256(void *mac, const struct test256EIA3_vectors_t *vector, #ifdef __aarch64__ int validate_zuc256_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, uint8_t *pDstData, uint8_t *pKeys, uint8_t *pIV, - const unsigned int job_api) + const enum api_type type) { uint32_t i; int ret = 0; @@ -1257,20 +1409,20 @@ int validate_zuc256_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, for (i = 0; i < NUM_ZUC_256_EIA3_TESTS; i++) { vector = &test256EIA3_vectors[i]; - memcpy(pKeys, vector->CK, ZUC256_KEY_LEN_IN_BYTES); + memcpy(pKeys, vector->CK, IMB_ZUC256_KEY_LEN_IN_BYTES); memcpy(pIV, vector->IV, vector->iv_length); bitLength = vector->length_in_bits; byteLength = (bitLength + 7) / 8; memcpy(pSrcData, vector->message, byteLength); iv_len = vector->iv_length; for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eia3_jobs(mb_mgr, &pKeys, &pIV, &pSrcData, &pDstData, &bitLength, 1, - ZUC256_KEY_LEN_IN_BYTES, tag_sz, + IMB_ZUC256_KEY_LEN_IN_BYTES, tag_sz, &iv_len); - else + else if (type == TEST_DIRECT_API) IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, pKeys, pIV, iv_len, pSrcData, bitLength, (uint32_t *)pDstData, tag_sz); @@ -1307,15 +1459,9 @@ int validate_zuc256_EIA_1_block(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, }; #endif -#ifdef __aarch64__ -int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, - uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs, const unsigned int job_api) -#else int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, uint8_t **pDstData, uint8_t **pKeys, uint8_t **pIV, - uint32_t numBuffs) -#endif + uint32_t numBuffs, const enum api_type type) { uint32_t i, j; int retTmp, ret = 0; @@ -1328,7 +1474,7 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (i = 0; i < NUM_ZUC_256_EIA3_TESTS; i++) { vector = &test256EIA3_vectors[i]; for (j = 0; j < numBuffs; j++) { - memcpy(pKeys[j], vector->CK, ZUC256_KEY_LEN_IN_BYTES); + memcpy(pKeys[j], vector->CK, IMB_ZUC256_KEY_LEN_IN_BYTES); memcpy(pIV[j], vector->IV, vector->iv_length); bitLength[j] = vector->length_in_bits; byteLength = (bitLength[j] + 7) / 8; @@ -1336,14 +1482,14 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, iv_lens[j] = vector->iv_length; } for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { -#ifdef __aarch64__ - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eia3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, bitLength, numBuffs, - ZUC256_KEY_LEN_IN_BYTES, tag_sz, - iv_lens); - else + IMB_ZUC256_KEY_LEN_IN_BYTES, + tag_sz, iv_lens); +#ifdef __aarch64__ + else if (type == TEST_DIRECT_API) IMB_ZUC256_EIA3_N_BUFFER(mb_mgr, (const void * const *)pKeys, (const void * const *)pIV, @@ -1351,13 +1497,13 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, (const void * const *)pSrcData, bitLength, (uint32_t **)pDstData, tag_sz, numBuffs); -#else - submit_eia3_jobs(mb_mgr, pKeys, pIV, - pSrcData, pDstData, - bitLength, numBuffs, - ZUC256_KEY_LEN_IN_BYTES, tag_sz, - iv_lens); #endif + else /* TEST_BURST_JOB_API */ + submit_burst_eia3_jobs(mb_mgr, pKeys, pIV, + pSrcData, pDstData, + bitLength, numBuffs, + IMB_ZUC256_KEY_LEN_IN_BYTES, + tag_sz, iv_lens); for (j = 0; j < numBuffs; j++) { retTmp = verify_tag_256(pDstData[j], vector, @@ -1372,7 +1518,7 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, * grouping all available tests vectors in groups of N buffers */ for (i = 0; i < numBuffs; i++) { vector = &test256EIA3_vectors[i % NUM_ZUC_256_EIA3_TESTS]; - memcpy(pKeys[i], vector->CK, ZUC256_KEY_LEN_IN_BYTES); + memcpy(pKeys[i], vector->CK, IMB_ZUC256_KEY_LEN_IN_BYTES); memcpy(pIV[i], vector->IV, vector->iv_length); bitLength[i] = vector->length_in_bits; @@ -1382,14 +1528,14 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, } for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { -#ifdef __aarch64__ - if (job_api) + if (type == TEST_SINGLE_JOB_API) submit_eia3_jobs(mb_mgr, pKeys, pIV, - pSrcData, pDstData, - bitLength, numBuffs, - ZUC256_KEY_LEN_IN_BYTES, tag_sz, - iv_lens); - else + pSrcData, pDstData, + bitLength, numBuffs, + IMB_ZUC256_KEY_LEN_IN_BYTES, + tag_sz, iv_lens); +#ifdef __aarch64__ + else if (type == TEST_DIRECT_API) IMB_ZUC256_EIA3_N_BUFFER(mb_mgr, (const void * const *)pKeys, (const void * const *)pIV, @@ -1397,13 +1543,13 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, (const void * const *)pSrcData, bitLength, (uint32_t **)pDstData, tag_sz, numBuffs); -#else - submit_eia3_jobs(mb_mgr, pKeys, pIV, - pSrcData, pDstData, - bitLength, numBuffs, - ZUC256_KEY_LEN_IN_BYTES, tag_sz, - iv_lens); #endif + else /* TEST_BURST_JOB_API */ + submit_burst_eia3_jobs(mb_mgr, pKeys, pIV, + pSrcData, pDstData, + bitLength, numBuffs, + IMB_ZUC256_KEY_LEN_IN_BYTES, + tag_sz, iv_lens); for (i = 0; i < numBuffs; i++) { const uint32_t vector_idx = i % NUM_ZUC_256_EIA3_TESTS; @@ -1429,8 +1575,8 @@ int validate_zuc_algorithm(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, } swapBytes; for (i = 0; i < NUM_ZUC_ALG_TESTS; i++) { - memcpy(pKeys, testZUC_vectors[i].CK, ZUC_KEY_LEN_IN_BYTES); - memcpy(pIV, testZUC_vectors[i].IV, ZUC_IV_LEN_IN_BYTES); + memcpy(pKeys, testZUC_vectors[i].CK, IMB_ZUC_KEY_LEN_IN_BYTES); + memcpy(pIV, testZUC_vectors[i].IV, IMB_ZUC_IV_LEN_IN_BYTES); memset(pSrcData, 0, 8); IMB_ZUC_EEA3_1_BUFFER(mb_mgr, pKeys, pIV, pSrcData, pDstData, 8); @@ -1440,8 +1586,10 @@ int validate_zuc_algorithm(struct IMB_MGR *mb_mgr, uint8_t *pSrcData, if (ret) printf("ZUC 1 algorithm test %u: FAIL\n", i); #ifdef DEBUG - else - printf("ZUC 1 algorithm test %u: PASS\n", i); + else { + if (!quiet_mode) + printf("ZUC 1 algorithm test %u: PASS\n", i); + } #endif } return ret; diff --git a/test/zuc_test_vectors.h b/test/kat-app/zuc_test_vectors.h similarity index 99% rename from test/zuc_test_vectors.h rename to test/kat-app/zuc_test_vectors.h index 6e8d399930483084b0e918a5264ec5081fbb9263..cf2f8e91a26d06914422c2ab7ca191a8ca3c96cc 100644 --- a/test/zuc_test_vectors.h +++ b/test/kat-app/zuc_test_vectors.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2009-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -33,13 +33,12 @@ #define NUM_ZUC_ALG_TESTS 3 #define NUM_ZUC_EEA3_TESTS 5 #define NUM_ZUC_256_EEA3_TESTS 10 +#ifndef __aarch64__ +#define NUM_ZUC_256_EIA3_TESTS 12 +#else #define NUM_ZUC_256_EIA3_TESTS 15 +#endif #define NUM_ZUC_EIA3_TESTS 10 -#define ZUC_KEY_LEN_IN_BYTES 16 -#define ZUC_IV_LEN_IN_BYTES 16 -#define ZUC256_KEY_LEN_IN_BYTES 32 -#define ZUC256_IV_LEN_IN_BYTES 25 -#define ZUC_DIGEST_LEN 4 typedef struct testZUC_vectors_t { uint8_t CK[16]; @@ -1749,6 +1748,7 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { 0x23, 0x48, 0x4b, 0xcf, 0x2e, 0x70, 0xe9, 0x5b }, }, +#ifdef __aarch64__ /* Add 3 cases, whose message length is not N*8 */ { /* Test 13 */ @@ -1864,5 +1864,6 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { 0x9F, 0xDA, 0x15, 0x0F, 0x35, 0x4E, 0xE8, 0x26}, }, +#endif /* __aarch64__ */ }; #endif diff --git a/test/poly1305_test.c b/test/poly1305_test.c deleted file mode 100644 index 7245956b57ba0c4d315e328fc7d2dab5ad657c9a..0000000000000000000000000000000000000000 --- a/test/poly1305_test.c +++ /dev/null @@ -1,600 +0,0 @@ -/***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include -#include -#include - -#include -#include "utils.h" - -int poly1305_test(struct IMB_MGR *mb_mgr); - -/* - * Test vectors from RFC7539 https://tools.ietf.org/html/rfc7539 - */ - -/* 2.5.2 Poly1305 Example and Test Vector */ -static const uint8_t msg_vec0[] = { - 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x67, 0x72, - 0x61, 0x70, 0x68, 0x69, 0x63, 0x20, 0x46, 0x6f, - 0x72, 0x75, 0x6d, 0x20, 0x52, 0x65, 0x73, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x20, 0x47, 0x72, 0x6f, - 0x75, 0x70 -}; - -static const uint8_t key_vec0[32] = { - 0x85, 0xd6, 0xbe, 0x78, 0x57, 0x55, 0x6d, 0x33, - 0x7f, 0x44, 0x52, 0xfe, 0x42, 0xd5, 0x06, 0xa8, - 0x01, 0x03, 0x80, 0x8a, 0xfb, 0x0d, 0xb2, 0xfd, - 0x4a, 0xbf, 0xf6, 0xaf, 0x41, 0x49, 0xf5, 0x1b -}; - -static const uint8_t tag_vec0[16] = { - 0xa8, 0x06, 0x1d, 0xc1, 0x30, 0x51, 0x36, 0xc6, - 0xc2, 0x2b, 0x8b, 0xaf, 0x0c, 0x01, 0x27, 0xa9 -}; - -/* A.3. Poly1305 Message Authentication Code */ - -/* Test Vector #1 */ -static const uint8_t msg_vec1[] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; -static const uint8_t key_vec1[32] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; -static const uint8_t tag_vec1[16] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -/* Test Vector #2 */ -static const uint8_t key_vec2[32] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x36, 0xe5, 0xf6, 0xb5, 0xc5, 0xe0, 0x60, 0x70, - 0xf0, 0xef, 0xca, 0x96, 0x22, 0x7a, 0x86, 0x3e -}; - -static const uint8_t msg_vec2[] = { - 0x41, 0x6e, 0x79, 0x20, 0x73, 0x75, 0x62, 0x6d, - 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x74, - 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x49, 0x45, - 0x54, 0x46, 0x20, 0x69, 0x6e, 0x74, 0x65, 0x6e, - 0x64, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x74, - 0x68, 0x65, 0x20, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x6f, 0x72, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x70, 0x75, 0x62, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x61, - 0x73, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x6f, 0x72, - 0x20, 0x70, 0x61, 0x72, 0x74, 0x20, 0x6f, 0x66, - 0x20, 0x61, 0x6e, 0x20, 0x49, 0x45, 0x54, 0x46, - 0x20, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, - 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x20, - 0x6f, 0x72, 0x20, 0x52, 0x46, 0x43, 0x20, 0x61, - 0x6e, 0x64, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x20, 0x6d, 0x61, 0x64, 0x65, 0x20, 0x77, 0x69, - 0x74, 0x68, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, - 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x20, 0x6f, 0x66, 0x20, 0x61, 0x6e, 0x20, 0x49, - 0x45, 0x54, 0x46, 0x20, 0x61, 0x63, 0x74, 0x69, - 0x76, 0x69, 0x74, 0x79, 0x20, 0x69, 0x73, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, - 0x65, 0x64, 0x20, 0x61, 0x6e, 0x20, 0x22, 0x49, - 0x45, 0x54, 0x46, 0x20, 0x43, 0x6f, 0x6e, 0x74, - 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x22, 0x2e, 0x20, 0x53, 0x75, 0x63, 0x68, 0x20, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x73, 0x20, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x20, 0x6f, 0x72, 0x61, 0x6c, 0x20, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x49, 0x45, - 0x54, 0x46, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x73, 0x2c, 0x20, 0x61, 0x73, 0x20, - 0x77, 0x65, 0x6c, 0x6c, 0x20, 0x61, 0x73, 0x20, - 0x77, 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, - 0x61, 0x6e, 0x64, 0x20, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x72, 0x6f, 0x6e, 0x69, 0x63, 0x20, 0x63, - 0x6f, 0x6d, 0x6d, 0x75, 0x6e, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x6d, 0x61, - 0x64, 0x65, 0x20, 0x61, 0x74, 0x20, 0x61, 0x6e, - 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x20, 0x6f, - 0x72, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x2c, - 0x20, 0x77, 0x68, 0x69, 0x63, 0x68, 0x20, 0x61, - 0x72, 0x65, 0x20, 0x61, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x65, 0x64, 0x20, 0x74, 0x6f -}; - -static const uint8_t tag_vec2[16] = { - 0x36, 0xe5, 0xf6, 0xb5, 0xc5, 0xe0, 0x60, 0x70, - 0xf0, 0xef, 0xca, 0x96, 0x22, 0x7a, 0x86, 0x3e -}; - -/* Test Vector #3 */ -static const uint8_t key_vec3[32] = { - 0x36, 0xe5, 0xf6, 0xb5, 0xc5, 0xe0, 0x60, 0x70, - 0xf0, 0xef, 0xca, 0x96, 0x22, 0x7a, 0x86, 0x3e, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -static const uint8_t msg_vec3[] = { - 0x41, 0x6e, 0x79, 0x20, 0x73, 0x75, 0x62, 0x6d, - 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x74, - 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x49, 0x45, - 0x54, 0x46, 0x20, 0x69, 0x6e, 0x74, 0x65, 0x6e, - 0x64, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x74, - 0x68, 0x65, 0x20, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x6f, 0x72, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x70, 0x75, 0x62, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x61, - 0x73, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x6f, 0x72, - 0x20, 0x70, 0x61, 0x72, 0x74, 0x20, 0x6f, 0x66, - 0x20, 0x61, 0x6e, 0x20, 0x49, 0x45, 0x54, 0x46, - 0x20, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, - 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x20, - 0x6f, 0x72, 0x20, 0x52, 0x46, 0x43, 0x20, 0x61, - 0x6e, 0x64, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x20, 0x6d, 0x61, 0x64, 0x65, 0x20, 0x77, 0x69, - 0x74, 0x68, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, - 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x20, 0x6f, 0x66, 0x20, 0x61, 0x6e, 0x20, 0x49, - 0x45, 0x54, 0x46, 0x20, 0x61, 0x63, 0x74, 0x69, - 0x76, 0x69, 0x74, 0x79, 0x20, 0x69, 0x73, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, - 0x65, 0x64, 0x20, 0x61, 0x6e, 0x20, 0x22, 0x49, - 0x45, 0x54, 0x46, 0x20, 0x43, 0x6f, 0x6e, 0x74, - 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x22, 0x2e, 0x20, 0x53, 0x75, 0x63, 0x68, 0x20, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x73, 0x20, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x20, 0x6f, 0x72, 0x61, 0x6c, 0x20, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x49, 0x45, - 0x54, 0x46, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x73, 0x2c, 0x20, 0x61, 0x73, 0x20, - 0x77, 0x65, 0x6c, 0x6c, 0x20, 0x61, 0x73, 0x20, - 0x77, 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, - 0x61, 0x6e, 0x64, 0x20, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x72, 0x6f, 0x6e, 0x69, 0x63, 0x20, 0x63, - 0x6f, 0x6d, 0x6d, 0x75, 0x6e, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x6d, 0x61, - 0x64, 0x65, 0x20, 0x61, 0x74, 0x20, 0x61, 0x6e, - 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x20, 0x6f, - 0x72, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x2c, - 0x20, 0x77, 0x68, 0x69, 0x63, 0x68, 0x20, 0x61, - 0x72, 0x65, 0x20, 0x61, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x65, 0x64, 0x20, 0x74, 0x6f -}; - -static const uint8_t tag_vec3[16] = { - 0xf3, 0x47, 0x7e, 0x7c, 0xd9, 0x54, 0x17, 0xaf, - 0x89, 0xa6, 0xb8, 0x79, 0x4c, 0x31, 0x0c, 0xf0 -}; - -/* Test Vector #4 */ -static const uint8_t key_vec4[32] = { - 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, - 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, - 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, - 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 -}; - -static const uint8_t msg_vec4[] = { - 0x27, 0x54, 0x77, 0x61, 0x73, 0x20, 0x62, 0x72, - 0x69, 0x6c, 0x6c, 0x69, 0x67, 0x2c, 0x20, 0x61, - 0x6e, 0x64, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, - 0x6c, 0x69, 0x74, 0x68, 0x79, 0x20, 0x74, 0x6f, - 0x76, 0x65, 0x73, 0x0a, 0x44, 0x69, 0x64, 0x20, - 0x67, 0x79, 0x72, 0x65, 0x20, 0x61, 0x6e, 0x64, - 0x20, 0x67, 0x69, 0x6d, 0x62, 0x6c, 0x65, 0x20, - 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x77, - 0x61, 0x62, 0x65, 0x3a, 0x0a, 0x41, 0x6c, 0x6c, - 0x20, 0x6d, 0x69, 0x6d, 0x73, 0x79, 0x20, 0x77, - 0x65, 0x72, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20, - 0x62, 0x6f, 0x72, 0x6f, 0x67, 0x6f, 0x76, 0x65, - 0x73, 0x2c, 0x0a, 0x41, 0x6e, 0x64, 0x20, 0x74, - 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x6d, 0x65, 0x20, - 0x72, 0x61, 0x74, 0x68, 0x73, 0x20, 0x6f, 0x75, - 0x74, 0x67, 0x72, 0x61, 0x62, 0x65, 0x2e -}; - -static const uint8_t tag_vec4[16] = { - 0x45, 0x41, 0x66, 0x9a, 0x7e, 0xaa, 0xee, 0x61, - 0xe7, 0x08, 0xdc, 0x7c, 0xbc, 0xc5, 0xeb, 0x62 -}; - -/* Test Vector #5 */ -static const uint8_t key_vec5[32] = { - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -static const uint8_t msg_vec5[] = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; - -static const uint8_t tag_vec5[16] = { - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -/* Test Vector #6 */ -static const uint8_t key_vec6[32] = { - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; - -static const uint8_t msg_vec6[] = { - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const uint8_t tag_vec6[16] = { - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -/* Test Vector #7 */ -static const uint8_t key_vec7[32] = { - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const uint8_t msg_vec7[] = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -static const uint8_t tag_vec7[16] = { - 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -/* Test Vector #8 */ -static const uint8_t key_vec8[32] = { - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -static const uint8_t msg_vec8[] = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xfb, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, - 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 -}; - -static const uint8_t tag_vec8[16] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -/* Test Vector #9 */ -static const uint8_t key_vec9[32] = { - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -static const uint8_t msg_vec9[] = { - 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; - -static const uint8_t tag_vec9[16] = { - 0xfa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; - -/* Test Vector #10 */ -static const uint8_t key_vec10[32] = { - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -static const uint8_t msg_vec10[] = { - 0xE3, 0x35, 0x94, 0xD7, 0x50, 0x5E, 0x43, 0xB9, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x33, 0x94, 0xD7, 0x50, 0x5E, 0x43, 0x79, 0xCD, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const uint8_t tag_vec10[16] = { - 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -/* Test Vector #11 */ -static const uint8_t key_vec11[32] = { - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -static const uint8_t msg_vec11[] = { - 0xE3, 0x35, 0x94, 0xD7, 0x50, 0x5E, 0x43, 0xB9, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x33, 0x94, 0xD7, 0x50, 0x5E, 0x43, 0x79, 0xCD, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const uint8_t tag_vec11[16] = { - 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -struct poly1305_vector { - const uint8_t *msg; - size_t msg_len; - const uint8_t *key; - const uint8_t *tag; -} rfc7539_vectors[] = { - {msg_vec0, sizeof(msg_vec0), key_vec0, tag_vec0}, - {msg_vec1, sizeof(msg_vec1), key_vec1, tag_vec1}, - {msg_vec2, sizeof(msg_vec2), key_vec2, tag_vec2}, - {msg_vec3, sizeof(msg_vec3), key_vec3, tag_vec3}, - {msg_vec4, sizeof(msg_vec4), key_vec4, tag_vec4}, - {msg_vec5, sizeof(msg_vec5), key_vec5, tag_vec5}, - {msg_vec6, sizeof(msg_vec6), key_vec6, tag_vec6}, - {msg_vec7, sizeof(msg_vec7), key_vec7, tag_vec7}, - {msg_vec8, sizeof(msg_vec8), key_vec8, tag_vec8}, - {msg_vec9, sizeof(msg_vec9), key_vec9, tag_vec9}, - {msg_vec10, sizeof(msg_vec10), key_vec10, tag_vec10}, - {msg_vec11, sizeof(msg_vec11), key_vec11, tag_vec11}, -}; - -static int -poly1305_job_ok(struct IMB_MGR *mb_mgr, - const struct poly1305_vector *vec, - const struct IMB_JOB *job, - const uint8_t *auth, - const uint8_t *padding, - const size_t sizeof_padding) -{ - const size_t auth_len = job->auth_tag_output_len_in_bytes; - - if (job->status != IMB_STATUS_COMPLETED) { - const int errcode = imb_get_errno(mb_mgr); - - printf("Error!: job status %d, errno %d => %s\n", - job->status, errcode, imb_get_strerror(errcode)); - return 0; - } - - /* hash checks */ - if (memcmp(padding, &auth[sizeof_padding + auth_len], - sizeof_padding)) { - printf("hash overwrite tail\n"); - hexdump(stderr, "Target", - &auth[sizeof_padding + auth_len], sizeof_padding); - return 0; - } - - if (memcmp(padding, &auth[0], sizeof_padding)) { - printf("hash overwrite head\n"); - hexdump(stderr, "Target", &auth[0], sizeof_padding); - return 0; - } - - if (memcmp(vec->tag, &auth[sizeof_padding], auth_len)) { - printf("hash mismatched\n"); - hexdump(stderr, "Received", &auth[sizeof_padding], - auth_len); - hexdump(stderr, "Expected", vec->tag, - auth_len); - return 0; - } - return 1; -} - -static int -test_poly1305(struct IMB_MGR *mb_mgr, - const struct poly1305_vector *vec, - const int dir, - const int num_jobs) -{ - struct IMB_JOB *job; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - int i = 0, jobs_rx = 0, ret = -1; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - auths[i] = malloc(16 + (sizeof(padding) * 2)); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - - memset(auths[i], -1, 16 + (sizeof(padding) * 2)); - } - - - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - /** - * Submit all jobs then flush any outstanding jobs - */ - for (i = 0; i < num_jobs; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - job->cipher_direction = dir; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_POLY1305; - - job->u.POLY1305._key = vec->key; - job->src = vec->msg; - job->msg_len_to_hash_in_bytes = vec->msg_len; - job->hash_start_src_offset_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = 16; - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job) { - jobs_rx++; - if (!poly1305_job_ok(mb_mgr, vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - jobs_rx++; - - if (!poly1305_job_ok(mb_mgr, vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - - if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); - goto end; - } - - - ret = 0; - - end: - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static void -test_poly1305_vectors(struct IMB_MGR *mb_mgr, - const int num_jobs, - const struct poly1305_vector *vec_array, - const size_t vec_array_size, - struct test_suite_context *ctx, - const char *banner) -{ - size_t vect; - - printf("%s (N jobs = %d):\n", banner, num_jobs); - for (vect = 0; vect < vec_array_size; vect++) { -#ifdef DEBUG - printf("Vector [%d/%d], M len: %d\n", - (int) vect + 1, (int) vec_array_size, - (int) vec_array[vect].msg_len); -#else - printf("."); -#endif - - if (test_poly1305(mb_mgr, &vec_array[vect], - IMB_DIR_ENCRYPT, num_jobs)) { - printf("error #%d encrypt\n", (int) vect + 1); - test_suite_update(ctx, 0, 1); - } else - test_suite_update(ctx, 1, 0); - - if (test_poly1305(mb_mgr, &vec_array[vect], - IMB_DIR_DECRYPT, num_jobs)) { - printf("error #%d decrypt\n", (int) vect + 1); - test_suite_update(ctx, 0, 1); - } else - test_suite_update(ctx, 1, 0); - } - printf("\n"); -} - - -int -poly1305_test(struct IMB_MGR *mb_mgr) -{ - struct test_suite_context ctx; - int i, errors; - - test_suite_start(&ctx, "POLY1305"); - for (i = 1; i < 20; i++) - test_poly1305_vectors(mb_mgr, i, rfc7539_vectors, - DIM(rfc7539_vectors), - &ctx, - "Poly1305 RFC7539 vectors"); - - errors = test_suite_end(&ctx); - - return errors; -} diff --git a/test/sha_test.c b/test/sha_test.c deleted file mode 100644 index a96b871f1be40e0d5217e303b3d8f168772b6d81..0000000000000000000000000000000000000000 --- a/test/sha_test.c +++ /dev/null @@ -1,618 +0,0 @@ -/***************************************************************************** - Copyright (c) 2018-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include -#include -#include - -#include -#include "gcm_ctr_vectors_test.h" -#include "utils.h" - -int sha_test(struct IMB_MGR *mb_mgr); - -/* - * Test vectors come from this NIST document: - * - * https://csrc.nist.gov/csrc/media/projects/ - * cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - */ -static const char message1[] = "abc"; -#define message1_len 3 - -static const char message2[] = ""; -#define message2_len 0 - -static const char message3[] = - "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; -#define message3_len 56 - -static const char message4[] = - "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmn" - "opjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; -#define message4_len 112 - -/* macro converts one 32-bit word into four 8-bit word */ -#define CONVERT_UINT32_TO_4xUINT8(v) \ - (((v) >> 24) & 0xff), (((v) >> 16) & 0xff), \ - (((v) >> 8) & 0xff), (((v) >> 0) & 0xff) - -/* macro converts one 64-bit word into eight 8-bit word */ -#define CONVERT_UINT64_TO_8xUINT8(v) \ - (((v) >> 56) & 0xff), (((v) >> 48) & 0xff), \ - (((v) >> 40) & 0xff), (((v) >> 32) & 0xff), \ - (((v) >> 24) & 0xff), (((v) >> 16) & 0xff), \ - (((v) >> 8) & 0xff), (((v) >> 0) & 0xff) - -static const char test_case1[] = "SHA-1 MSG1"; -#define data1 ((const uint8_t *)message1) -#define data_len1 message1_len -static const uint8_t digest1[] = { - /* a9993e36 4706816a ba3e2571 7850c26c 9cd0d89d */ - CONVERT_UINT32_TO_4xUINT8(0xa9993e36), - CONVERT_UINT32_TO_4xUINT8(0x4706816a), - CONVERT_UINT32_TO_4xUINT8(0xba3e2571), - CONVERT_UINT32_TO_4xUINT8(0x7850c26c), - CONVERT_UINT32_TO_4xUINT8(0x9cd0d89d) -}; -#define digest_len1 sizeof(digest1) - -static const char test_case2[] = "SHA-224 MSG1"; -#define data2 ((const uint8_t *)message1) -#define data_len2 message1_len -static const uint8_t digest2[] = { - /* 23097d22 3405d822 8642a477 bda255b3 */ - /* 2aadbce4 bda0b3f7 e36c9da7 */ - CONVERT_UINT32_TO_4xUINT8(0x23097d22), - CONVERT_UINT32_TO_4xUINT8(0x3405d822), - CONVERT_UINT32_TO_4xUINT8(0x8642a477), - CONVERT_UINT32_TO_4xUINT8(0xbda255b3), - CONVERT_UINT32_TO_4xUINT8(0x2aadbce4), - CONVERT_UINT32_TO_4xUINT8(0xbda0b3f7), - CONVERT_UINT32_TO_4xUINT8(0xe36c9da7) -}; -#define digest_len2 sizeof(digest2) - -static const char test_case3[] = "SHA-256 MSG1"; -#define data3 ((const uint8_t *)message1) -#define data_len3 message1_len -static const uint8_t digest3[] = { - /* ba7816bf 8f01cfea 414140de 5dae2223 */ - /* b00361a3 96177a9c b410ff61 f20015ad */ - CONVERT_UINT32_TO_4xUINT8(0xba7816bf), - CONVERT_UINT32_TO_4xUINT8(0x8f01cfea), - CONVERT_UINT32_TO_4xUINT8(0x414140de), - CONVERT_UINT32_TO_4xUINT8(0x5dae2223), - CONVERT_UINT32_TO_4xUINT8(0xb00361a3), - CONVERT_UINT32_TO_4xUINT8(0x96177a9c), - CONVERT_UINT32_TO_4xUINT8(0xb410ff61), - CONVERT_UINT32_TO_4xUINT8(0xf20015ad) -}; -#define digest_len3 sizeof(digest3) - -static const char test_case4[] = "SHA-384 MSG1"; -#define data4 ((const uint8_t *)message1) -#define data_len4 message1_len -static const uint8_t digest4[] = { - /* cb00753f45a35e8b b5a03d699ac65007 */ - /* 272c32ab0eded163 1a8b605a43ff5bed */ - /* 8086072ba1e7cc23 58baeca134c825a7 */ - CONVERT_UINT64_TO_8xUINT8(0xcb00753f45a35e8b), - CONVERT_UINT64_TO_8xUINT8(0xb5a03d699ac65007), - CONVERT_UINT64_TO_8xUINT8(0x272c32ab0eded163), - CONVERT_UINT64_TO_8xUINT8(0x1a8b605a43ff5bed), - CONVERT_UINT64_TO_8xUINT8(0x8086072ba1e7cc23), - CONVERT_UINT64_TO_8xUINT8(0x58baeca134c825a7) -}; -#define digest_len4 sizeof(digest4) - -static const char test_case5[] = "SHA-512 MSG1"; -#define data5 ((const uint8_t *)message1) -#define data_len5 message1_len -static const uint8_t digest5[] = { - /* ddaf35a193617aba cc417349ae204131 */ - /* 12e6fa4e89a97ea2 0a9eeee64b55d39a */ - /* 2192992a274fc1a8 36ba3c23a3feebbd */ - /* 454d4423643ce80e 2a9ac94fa54ca49f */ - CONVERT_UINT64_TO_8xUINT8(0xddaf35a193617aba), - CONVERT_UINT64_TO_8xUINT8(0xcc417349ae204131), - CONVERT_UINT64_TO_8xUINT8(0x12e6fa4e89a97ea2), - CONVERT_UINT64_TO_8xUINT8(0x0a9eeee64b55d39a), - CONVERT_UINT64_TO_8xUINT8(0x2192992a274fc1a8), - CONVERT_UINT64_TO_8xUINT8(0x36ba3c23a3feebbd), - CONVERT_UINT64_TO_8xUINT8(0x454d4423643ce80e), - CONVERT_UINT64_TO_8xUINT8(0x2a9ac94fa54ca49f) -}; -#define digest_len5 sizeof(digest5) - -static const char test_case10[] = "SHA-1 MSG2"; -#define data10 ((const uint8_t *)message2) -#define data_len10 message2_len -static const uint8_t digest10[] = { - CONVERT_UINT32_TO_4xUINT8(0xda39a3ee), - CONVERT_UINT32_TO_4xUINT8(0x5e6b4b0d), - CONVERT_UINT32_TO_4xUINT8(0x3255bfef), - CONVERT_UINT32_TO_4xUINT8(0x95601890), - CONVERT_UINT32_TO_4xUINT8(0xafd80709) -}; -#define digest_len10 sizeof(digest10) - -static const char test_case11[] = "SHA-224 MSG2"; -#define data11 ((const uint8_t *)message2) -#define data_len11 message2_len -static const uint8_t digest11[] = { - CONVERT_UINT32_TO_4xUINT8(0xd14a028c), - CONVERT_UINT32_TO_4xUINT8(0x2a3a2bc9), - CONVERT_UINT32_TO_4xUINT8(0x476102bb), - CONVERT_UINT32_TO_4xUINT8(0x288234c4), - CONVERT_UINT32_TO_4xUINT8(0x15a2b01f), - CONVERT_UINT32_TO_4xUINT8(0x828ea62a), - CONVERT_UINT32_TO_4xUINT8(0xc5b3e42f) -}; -#define digest_len11 sizeof(digest11) - -static const char test_case12[] = "SHA-256 MSG2"; -#define data12 ((const uint8_t *)message2) -#define data_len12 message2_len -static const uint8_t digest12[] = { - CONVERT_UINT32_TO_4xUINT8(0xe3b0c442), - CONVERT_UINT32_TO_4xUINT8(0x98fc1c14), - CONVERT_UINT32_TO_4xUINT8(0x9afbf4c8), - CONVERT_UINT32_TO_4xUINT8(0x996fb924), - CONVERT_UINT32_TO_4xUINT8(0x27ae41e4), - CONVERT_UINT32_TO_4xUINT8(0x649b934c), - CONVERT_UINT32_TO_4xUINT8(0xa495991b), - CONVERT_UINT32_TO_4xUINT8(0x7852b855) -}; -#define digest_len12 sizeof(digest12) - -static const char test_case13[] = "SHA-384 MSG2"; -#define data13 ((const uint8_t *)message2) -#define data_len13 message2_len -static const uint8_t digest13[] = { - CONVERT_UINT64_TO_8xUINT8(0x38b060a751ac9638), - CONVERT_UINT64_TO_8xUINT8(0x4cd9327eb1b1e36a), - CONVERT_UINT64_TO_8xUINT8(0x21fdb71114be0743), - CONVERT_UINT64_TO_8xUINT8(0x4c0cc7bf63f6e1da), - CONVERT_UINT64_TO_8xUINT8(0x274edebfe76f65fb), - CONVERT_UINT64_TO_8xUINT8(0xd51ad2f14898b95b) -}; -#define digest_len13 sizeof(digest13) - -static const char test_case14[] = "SHA-512 MSG2"; -#define data14 ((const uint8_t *)message2) -#define data_len14 message2_len -static const uint8_t digest14[] = { - CONVERT_UINT64_TO_8xUINT8(0xcf83e1357eefb8bd), - CONVERT_UINT64_TO_8xUINT8(0xf1542850d66d8007), - CONVERT_UINT64_TO_8xUINT8(0xd620e4050b5715dc), - CONVERT_UINT64_TO_8xUINT8(0x83f4a921d36ce9ce), - CONVERT_UINT64_TO_8xUINT8(0x47d0d13c5d85f2b0), - CONVERT_UINT64_TO_8xUINT8(0xff8318d2877eec2f), - CONVERT_UINT64_TO_8xUINT8(0x63b931bd47417a81), - CONVERT_UINT64_TO_8xUINT8(0xa538327af927da3e) -}; -#define digest_len14 sizeof(digest14) - -static const char test_case20[] = "SHA-1 MSG3"; -#define data20 ((const uint8_t *)message3) -#define data_len20 message3_len -static const uint8_t digest20[] = { - CONVERT_UINT32_TO_4xUINT8(0x84983e44), - CONVERT_UINT32_TO_4xUINT8(0x1c3bd26e), - CONVERT_UINT32_TO_4xUINT8(0xbaae4aa1), - CONVERT_UINT32_TO_4xUINT8(0xf95129e5), - CONVERT_UINT32_TO_4xUINT8(0xe54670f1) -}; -#define digest_len20 sizeof(digest20) - -static const char test_case21[] = "SHA-224 MSG3"; -#define data21 ((const uint8_t *)message3) -#define data_len21 message3_len -static const uint8_t digest21[] = { - CONVERT_UINT32_TO_4xUINT8(0x75388b16), - CONVERT_UINT32_TO_4xUINT8(0x512776cc), - CONVERT_UINT32_TO_4xUINT8(0x5dba5da1), - CONVERT_UINT32_TO_4xUINT8(0xfd890150), - CONVERT_UINT32_TO_4xUINT8(0xb0c6455c), - CONVERT_UINT32_TO_4xUINT8(0xb4f58b19), - CONVERT_UINT32_TO_4xUINT8(0x52522525) -}; -#define digest_len21 sizeof(digest21) - -static const char test_case22[] = "SHA-256 MSG3"; -#define data22 ((const uint8_t *)message3) -#define data_len22 message3_len -static const uint8_t digest22[] = { - CONVERT_UINT32_TO_4xUINT8(0x248d6a61), - CONVERT_UINT32_TO_4xUINT8(0xd20638b8), - CONVERT_UINT32_TO_4xUINT8(0xe5c02693), - CONVERT_UINT32_TO_4xUINT8(0x0c3e6039), - CONVERT_UINT32_TO_4xUINT8(0xa33ce459), - CONVERT_UINT32_TO_4xUINT8(0x64ff2167), - CONVERT_UINT32_TO_4xUINT8(0xf6ecedd4), - CONVERT_UINT32_TO_4xUINT8(0x19db06c1) -}; -#define digest_len22 sizeof(digest22) - -static const char test_case23[] = "SHA-384 MSG3"; -#define data23 ((const uint8_t *)message3) -#define data_len23 message3_len -static const uint8_t digest23[] = { - CONVERT_UINT64_TO_8xUINT8(0x3391fdddfc8dc739), - CONVERT_UINT64_TO_8xUINT8(0x3707a65b1b470939), - CONVERT_UINT64_TO_8xUINT8(0x7cf8b1d162af05ab), - CONVERT_UINT64_TO_8xUINT8(0xfe8f450de5f36bc6), - CONVERT_UINT64_TO_8xUINT8(0xb0455a8520bc4e6f), - CONVERT_UINT64_TO_8xUINT8(0x5fe95b1fe3c8452b) -}; -#define digest_len23 sizeof(digest23) - -static const char test_case24[] = "SHA-512 MSG3"; -#define data24 ((const uint8_t *)message3) -#define data_len24 message3_len -static const uint8_t digest24[] = { - CONVERT_UINT64_TO_8xUINT8(0x204a8fc6dda82f0a), - CONVERT_UINT64_TO_8xUINT8(0x0ced7beb8e08a416), - CONVERT_UINT64_TO_8xUINT8(0x57c16ef468b228a8), - CONVERT_UINT64_TO_8xUINT8(0x279be331a703c335), - CONVERT_UINT64_TO_8xUINT8(0x96fd15c13b1b07f9), - CONVERT_UINT64_TO_8xUINT8(0xaa1d3bea57789ca0), - CONVERT_UINT64_TO_8xUINT8(0x31ad85c7a71dd703), - CONVERT_UINT64_TO_8xUINT8(0x54ec631238ca3445) -}; -#define digest_len24 sizeof(digest24) - -static const char test_case30[] = "SHA-1 MSG4"; -#define data30 ((const uint8_t *)message4) -#define data_len30 message4_len -static const uint8_t digest30[] = { - CONVERT_UINT32_TO_4xUINT8(0xa49b2446), - CONVERT_UINT32_TO_4xUINT8(0xa02c645b), - CONVERT_UINT32_TO_4xUINT8(0xf419f995), - CONVERT_UINT32_TO_4xUINT8(0xb6709125), - CONVERT_UINT32_TO_4xUINT8(0x3a04a259) -}; -#define digest_len30 sizeof(digest30) - -static const char test_case31[] = "SHA-224 MSG4"; -#define data31 ((const uint8_t *)message4) -#define data_len31 message4_len -static const uint8_t digest31[] = { - CONVERT_UINT32_TO_4xUINT8(0xc97ca9a5), - CONVERT_UINT32_TO_4xUINT8(0x59850ce9), - CONVERT_UINT32_TO_4xUINT8(0x7a04a96d), - CONVERT_UINT32_TO_4xUINT8(0xef6d99a9), - CONVERT_UINT32_TO_4xUINT8(0xe0e0e2ab), - CONVERT_UINT32_TO_4xUINT8(0x14e6b8df), - CONVERT_UINT32_TO_4xUINT8(0x265fc0b3) -}; -#define digest_len31 sizeof(digest31) - -static const char test_case32[] = "SHA-256 MSG4"; -#define data32 ((const uint8_t *)message4) -#define data_len32 message4_len -static const uint8_t digest32[] = { - CONVERT_UINT32_TO_4xUINT8(0xcf5b16a7), - CONVERT_UINT32_TO_4xUINT8(0x78af8380), - CONVERT_UINT32_TO_4xUINT8(0x036ce59e), - CONVERT_UINT32_TO_4xUINT8(0x7b049237), - CONVERT_UINT32_TO_4xUINT8(0x0b249b11), - CONVERT_UINT32_TO_4xUINT8(0xe8f07a51), - CONVERT_UINT32_TO_4xUINT8(0xafac4503), - CONVERT_UINT32_TO_4xUINT8(0x7afee9d1) -}; -#define digest_len32 sizeof(digest32) - -static const char test_case33[] = "SHA-384 MSG4"; -#define data33 ((const uint8_t *)message4) -#define data_len33 message4_len -static const uint8_t digest33[] = { - CONVERT_UINT64_TO_8xUINT8(0x09330c33f71147e8), - CONVERT_UINT64_TO_8xUINT8(0x3d192fc782cd1b47), - CONVERT_UINT64_TO_8xUINT8(0x53111b173b3b05d2), - CONVERT_UINT64_TO_8xUINT8(0x2fa08086e3b0f712), - CONVERT_UINT64_TO_8xUINT8(0xfcc7c71a557e2db9), - CONVERT_UINT64_TO_8xUINT8(0x66c3e9fa91746039) -}; -#define digest_len33 sizeof(digest33) - -static const char test_case34[] = "SHA-512 MSG4"; -#define data34 ((const uint8_t *)message4) -#define data_len34 message4_len -static const uint8_t digest34[] = { - CONVERT_UINT64_TO_8xUINT8(0x8e959b75dae313da), - CONVERT_UINT64_TO_8xUINT8(0x8cf4f72814fc143f), - CONVERT_UINT64_TO_8xUINT8(0x8f7779c6eb9f7fa1), - CONVERT_UINT64_TO_8xUINT8(0x7299aeadb6889018), - CONVERT_UINT64_TO_8xUINT8(0x501d289e4900f7e4), - CONVERT_UINT64_TO_8xUINT8(0x331b99dec4b5433a), - CONVERT_UINT64_TO_8xUINT8(0xc7d329eeb6dd2654), - CONVERT_UINT64_TO_8xUINT8(0x5e96e55b874be909) -}; -#define digest_len34 sizeof(digest34) - -#define SHA_TEST_VEC(num, size) \ - { test_case##num, size, \ - (const uint8_t *) data##num, data_len##num, \ - (const uint8_t *) digest##num, digest_len##num } - -static const struct sha_vector { - const char *test_case; - int sha_type; /* 1, 224, 256, 384 or 512 */ - const uint8_t *data; - size_t data_len; - const uint8_t *digest; - size_t digest_len; -} sha_vectors[] = { - SHA_TEST_VEC(1, 1), - SHA_TEST_VEC(2, 224), - SHA_TEST_VEC(3, 256), - SHA_TEST_VEC(4, 384), - SHA_TEST_VEC(5, 512), - SHA_TEST_VEC(10, 1), - SHA_TEST_VEC(11, 224), - SHA_TEST_VEC(12, 256), - SHA_TEST_VEC(13, 384), - SHA_TEST_VEC(14, 512), - SHA_TEST_VEC(20, 1), - SHA_TEST_VEC(21, 224), - SHA_TEST_VEC(22, 256), - SHA_TEST_VEC(23, 384), - SHA_TEST_VEC(24, 512), - SHA_TEST_VEC(30, 1), - SHA_TEST_VEC(31, 224), - SHA_TEST_VEC(32, 256), - SHA_TEST_VEC(33, 384), - SHA_TEST_VEC(34, 512) -}; - -static int -sha_job_ok(const struct sha_vector *vec, - const struct IMB_JOB *job, - const uint8_t *auth, - const uint8_t *padding, - const size_t sizeof_padding) -{ - if (job->status != IMB_STATUS_COMPLETED) { - printf("line:%d job error status:%d ", __LINE__, job->status); - return 0; - } - - /* hash checks */ - if (memcmp(padding, &auth[sizeof_padding + vec->digest_len], - sizeof_padding)) { - printf("hash overwrite tail\n"); - hexdump(stderr, "Target", - &auth[sizeof_padding + vec->digest_len], - sizeof_padding); - return 0; - } - - if (memcmp(padding, &auth[0], sizeof_padding)) { - printf("hash overwrite head\n"); - hexdump(stderr, "Target", &auth[0], sizeof_padding); - return 0; - } - - if (memcmp(vec->digest, &auth[sizeof_padding], - vec->digest_len)) { - printf("hash mismatched\n"); - hexdump(stderr, "Received", &auth[sizeof_padding], - vec->digest_len); - hexdump(stderr, "Expected", vec->digest, - vec->digest_len); - return 0; - } - return 1; -} - -static int -test_sha(struct IMB_MGR *mb_mgr, - const struct sha_vector *vec, - const int num_jobs) -{ - struct IMB_JOB *job; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - int i = 0, jobs_rx = 0, ret = -1; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - const size_t alloc_len = - vec->digest_len + (sizeof(padding) * 2); - - auths[i] = malloc(alloc_len); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - memset(auths[i], -1, alloc_len); - } - - /* empty the manager */ - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - - memset(job, 0, sizeof(*job)); - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->digest_len; - job->src = vec->data; - job->msg_len_to_hash_in_bytes = vec->data_len; - job->cipher_mode = IMB_CIPHER_NULL; - switch (vec->sha_type) { - case 1: - job->hash_alg = IMB_AUTH_SHA_1; - break; - case 224: - job->hash_alg = IMB_AUTH_SHA_224; - break; - case 256: - job->hash_alg = IMB_AUTH_SHA_256; - break; - case 384: - job->hash_alg = IMB_AUTH_SHA_384; - break; - case 512: - default: - job->hash_alg = IMB_AUTH_SHA_512; - break; - } - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job) { - jobs_rx++; - if (!sha_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - jobs_rx++; - if (!sha_job_ok(vec, job, job->user_data, - padding, sizeof(padding))) - goto end; - } - - if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); - goto end; - } - ret = 0; - - end: - /* empty the manager before next tests */ - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static void -test_sha_vectors(struct IMB_MGR *mb_mgr, - struct test_suite_context *sha1_ctx, - struct test_suite_context *sha224_ctx, - struct test_suite_context *sha256_ctx, - struct test_suite_context *sha384_ctx, - struct test_suite_context *sha512_ctx, - const int num_jobs) -{ - const int vectors_cnt = - sizeof(sha_vectors) / sizeof(sha_vectors[0]); - int vect; - struct test_suite_context *ctx; - - printf("SHA standard test vectors (N jobs = %d):\n", num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("[%d/%d] SHA%d Test Case %s data_len:%d " - "digest_len:%d\n", - vect, vectors_cnt, - sha_vectors[idx].sha_type, - sha_vectors[idx].test_case, - (int) sha_vectors[idx].data_len, - (int) sha_vectors[idx].digest_len); -#endif - switch (sha_vectors[idx].sha_type) { - case 1: - ctx = sha1_ctx; - break; - case 224: - ctx = sha224_ctx; - break; - case 256: - ctx = sha256_ctx; - break; - case 384: - ctx = sha384_ctx; - break; - case 512: - default: - ctx = sha512_ctx; - break; - } - - if (test_sha(mb_mgr, &sha_vectors[idx], num_jobs)) { - printf("error #%d\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - } -} - -int -sha_test(struct IMB_MGR *mb_mgr) -{ - struct test_suite_context sha1_ctx, sha224_ctx, sha256_ctx; - struct test_suite_context sha384_ctx, sha512_ctx; - int errors; - unsigned i; - - test_suite_start(&sha1_ctx, "SHA1"); - test_suite_start(&sha224_ctx, "SHA224"); - test_suite_start(&sha256_ctx, "SHA256"); - test_suite_start(&sha384_ctx, "SHA384"); - test_suite_start(&sha512_ctx, "SHA512"); - for (i = 1; i <= 17; i++) { - test_sha_vectors(mb_mgr, &sha1_ctx, &sha224_ctx, - &sha256_ctx, &sha384_ctx, &sha512_ctx, i); - } - errors = test_suite_end(&sha1_ctx); - errors += test_suite_end(&sha224_ctx); - errors += test_suite_end(&sha256_ctx); - errors += test_suite_end(&sha384_ctx); - errors += test_suite_end(&sha512_ctx); - - return errors; -} diff --git a/test/win_x64.mak b/test/win_x64.mak index 896f9c49a36858cc921421cef19a36134c642d1e..761140f5132bbdd4c9d90e8c493692ace0d13aa0 100644 --- a/test/win_x64.mak +++ b/test/win_x64.mak @@ -1,5 +1,5 @@ # -# Copyright (c) 2017-2022, Intel Corporation +# Copyright (c) 2017-2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -25,82 +25,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TEST_APP = ipsec_MB_testapp -XVALID_APP = ipsec_xvalid_test -INSTNAME = ipsec-mb - -!if !defined(PREFIX) -PREFIX = C:\Program Files -!endif - -!if exist("$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib") -IPSECLIB = "$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib" -INCDIR = -I"$(PREFIX)\$(INSTNAME)" -!else -!if !defined(LIB_DIR) -LIB_DIR = ..\lib -!endif -IPSECLIB = "$(LIB_DIR)\libIPSec_MB.lib" -INCDIR = -I$(LIB_DIR) -I.\ -!endif - -!if !defined(DEBUG_OPT) -DEBUG_OPT = /Od -!endif - -!ifdef DEBUG -DCFLAGS = $(DEBUG_OPT) /DDEBUG /Z7 -DLFLAGS = /debug -!else -DCFLAGS = /O2 /Oi -DLFLAGS = -!endif - -# compiler -CC = cl - -# _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure snprintf() being used -CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) - -#linker -LNK = link -TEST_LFLAGS = /out:$(TEST_APP).exe $(DLFLAGS) -XVALID_LFLAGS = /out:$(XVALID_APP).exe $(DLFLAGS) - -AS = nasm -AFLAGS = -Werror -fwin64 -Xvc -DWIN_ABI - -# dependency -!ifndef DEPTOOL -DEPTOOL = ..\mkdep.bat -!endif -DEPFLAGS = $(INCDIR) - -TEST_OBJS = main.obj gcm_test.obj ctr_test.obj customop_test.obj des_test.obj ccm_test.obj cmac_test.obj hmac_sha1_test.obj hmac_sha256_sha512_test.obj utils.obj hmac_md5_test.obj aes_test.obj sha_test.obj chained_test.obj api_test.obj pon_test.obj ecb_test.obj zuc_test.obj kasumi_test.obj snow3g_test.obj direct_api_test.obj clear_mem_test.obj hec_test.obj xcbc_test.obj aes_cbcs_test.obj crc_test.obj chacha_test.obj poly1305_test.obj chacha20_poly1305_test.obj null_test.obj snow_v_test.obj direct_api_param_test.obj - -XVALID_OBJS = ipsec_xvalid.obj misc.obj utils.obj - -all: $(TEST_APP).exe $(XVALID_APP).exe tests.dep - -$(TEST_APP).exe: $(TEST_OBJS) $(IPSECLIB) - $(LNK) $(TEST_LFLAGS) $(TEST_OBJS) $(IPSECLIB) - -$(XVALID_APP).exe: $(XVALID_OBJS) $(IPSECLIB) - $(LNK) $(XVALID_LFLAGS) $(XVALID_OBJS) $(IPSECLIB) - -tests.dep: $(TEST_OBJS) $(XVALID_OBJS) - @type *.obj.dep > $@ 2> nul - -.c.obj: - $(CC) /c $(CFLAGS) $< - $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep - -.asm.obj: - $(AS) -MD $@.dep -o $@ $(AFLAGS) $< +all: + cd kat-app & $(MAKE) /f win_x64.mak + cd xvalid-app & $(MAKE) /f win_x64.mak + cd wycheproof-app & $(MAKE) /f win_x64.mak clean: - del /q $(TEST_OBJS) tests.dep *.obj.dep $(TEST_APP).* $(XVALID_OBJS) $(XVALID_APP).* - -!if exist(tests.dep) -!include tests.dep -!endif + cd kat-app & $(MAKE) /f win_x64.mak clean + cd xvalid-app & $(MAKE) /f win_x64.mak clean + cd wycheproof-app & $(MAKE) /f win_x64.mak clean diff --git a/test/wycheproof-app/CMakeLists.txt b/test/wycheproof-app/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..eb16016fc59ccfb5a6a6f12bcc973db8728d1fce --- /dev/null +++ b/test/wycheproof-app/CMakeLists.txt @@ -0,0 +1,84 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +######################################## +# set app and library names +######################################## +set(WYCHEPROOF_APP imb-wycheproof) +set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib) +set(TEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include) + +######################################## +# set imb-wycheproof source files +######################################## +set(WYCHEPROOF_APP_SRC_FILES_C + ${CMAKE_CURRENT_SOURCE_DIR}/aes_gcm_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/aes_ccm_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/chacha20_poly1305_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/aes_cmac_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/gmac_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/gmac_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha1_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha224_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha256_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha384_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/hmac_sha512_test.json.c + ${CMAKE_CURRENT_SOURCE_DIR}/wycheproof.c +) + +######################################## +# set C compiler and NASM options +######################################## +# add OS specific options +if(WINDOWS) + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/windows.cmake) +else() + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/unix.cmake) +endif() + + +######################################## +# add targets +######################################## +include_directories(${LIB_DIR} ${TEST_INCLUDE_DIR}) +link_directories(${LIB_DIR}) + +add_executable(${WYCHEPROOF_APP} ${WYCHEPROOF_APP_SRC_FILES_C}) +target_link_libraries(${WYCHEPROOF_APP} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${WYCHEPROOF_APP} PRIVATE ${APP_DEFINES}) + +######################################## +# add tests +######################################## + +add_test(NAME WYCHEPROOF COMMAND ${WYCHEPROOF_APP}) +add_test(NAME WYCHEPROOF-SHANI-OFF COMMAND ${WYCHEPROOF_APP} --shani-off) +add_test(NAME WYCHEPROOF-GFNI-OFF COMMAND ${WYCHEPROOF_APP} --gfni-off) + diff --git a/test/wycheproof-app/Makefile b/test/wycheproof-app/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..461e3f41a7639c0cbc1da871041ceea4cc99db56 --- /dev/null +++ b/test/wycheproof-app/Makefile @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP := imb-wycheproof + +# include common options +include ../common/common.mk + +OBJ_FILES = aes_gcm_test.json.o aes_ccm_test.json.o \ + chacha20_poly1305_test.json.o \ + aes_cmac_test.json.o gmac_test.json.o gmac_test.json.o \ + hmac_sha1_test.json.o hmac_sha224_test.json.o hmac_sha256_test.json.o \ + hmac_sha384_test.json.o hmac_sha512_test.json.o \ + wycheproof.o + +all: $(APP) + +$(APP): $(OBJ_FILES) + $(CC) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +.PHONY: clean +clean: + -rm -f $(OBJ_FILES) *.d $(APP) + +# if target not clean then make dependencies +ifneq ($(MAKECMDGOALS),clean) +-include $(wildcard *.d) +endif diff --git a/test/wycheproof-app/README.md b/test/wycheproof-app/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9a05c4ecffd533e9b87435aebfbd4f2c270f20d5 --- /dev/null +++ b/test/wycheproof-app/README.md @@ -0,0 +1,39 @@ +# Intel(R) Multi-Buffer Crypto for IPsec Library - Wycheproof Test Application + +## Contents + +- Overview +- Usage + + +## Overview + +The wycheproof test application verifies supported algorithm implementations against +[Project Wycheproof](https://github.com/google/wycheproof) test vectors. Project Wycheproof +provides a set of vectors to check for expected behaviors and detect known weaknesses for +specific algorithms. + +## Usage + +Before running the application, ensure the library is installed by following the instructions +in the [README](https://github.com/intel/intel-ipsec-mb/tree/main/test#library-installation). + +### Linux +To test all supported algorithms on all architectures: +`./imb-wycheproof` + +To test all supported algorithms on AVX512 architecture only: +`./imb-wycheproof --avx512` + +To display an extensive help page: +`./imb-wycheproof --help` + +### Windows +To test all supported algorithms on all architectures: +`imb-wycheproof.exe` + +To test all supported algorithms on AVX512 architecture only: +`imb-wycheproof.exe --avx512` + +To display an extensive help page: +`imb-wycheproof.exe --help` diff --git a/test/wycheproof/aes_ccm_test.json.c b/test/wycheproof-app/aes_ccm_test.json.c similarity index 99% rename from test/wycheproof/aes_ccm_test.json.c rename to test/wycheproof-app/aes_ccm_test.json.c index da8f1791939b3ad8bec1967617b9d19cdb4a1113..e32985d58fb4733dd15276ef9f09b2dfbcbead61 100644 --- a/test/wycheproof/aes_ccm_test.json.c +++ b/test/wycheproof-app/aes_ccm_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/aes_cmac_test.json.c b/test/wycheproof-app/aes_cmac_test.json.c similarity index 99% rename from test/wycheproof/aes_cmac_test.json.c rename to test/wycheproof-app/aes_cmac_test.json.c index 911556f0a324cc2d7bfb360055c207c0f3b39723..5e13976f30ee192a5387c48e9c1a8905760815dd 100644 --- a/test/wycheproof/aes_cmac_test.json.c +++ b/test/wycheproof-app/aes_cmac_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/aes_gcm_test.json.c b/test/wycheproof-app/aes_gcm_test.json.c similarity index 99% rename from test/wycheproof/aes_gcm_test.json.c rename to test/wycheproof-app/aes_gcm_test.json.c index bfa3a3d15ee95e90a47163f4354bb4b7bdafdbc2..3bc14666e2473ec3b2086fc59ea9a27651feddc5 100644 --- a/test/wycheproof/aes_gcm_test.json.c +++ b/test/wycheproof-app/aes_gcm_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/chacha20_poly1305_test.json.c b/test/wycheproof-app/chacha20_poly1305_test.json.c similarity index 99% rename from test/wycheproof/chacha20_poly1305_test.json.c rename to test/wycheproof-app/chacha20_poly1305_test.json.c index 9a30de045ba3f460e8bf11d1b03075d0968a2753..d78485cedb15f12ce9616f3baffd37fa70fb7e22 100644 --- a/test/wycheproof/chacha20_poly1305_test.json.c +++ b/test/wycheproof-app/chacha20_poly1305_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/gmac_test.json.c b/test/wycheproof-app/gmac_test.json.c similarity index 99% rename from test/wycheproof/gmac_test.json.c rename to test/wycheproof-app/gmac_test.json.c index 8730d33a6799f51f88ae850e5dc1e56887e95477..f15bd25919396c66395448a38f76aff4bd88b683 100644 --- a/test/wycheproof/gmac_test.json.c +++ b/test/wycheproof-app/gmac_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/hmac_sha1_test.json.c b/test/wycheproof-app/hmac_sha1_test.json.c similarity index 99% rename from test/wycheproof/hmac_sha1_test.json.c rename to test/wycheproof-app/hmac_sha1_test.json.c index d0c1d36edb07db6b9c5e310f870113c6d2a961f6..db246e7c9201d5701c6d40a0901970c7bb9b06ec 100644 --- a/test/wycheproof/hmac_sha1_test.json.c +++ b/test/wycheproof-app/hmac_sha1_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/hmac_sha224_test.json.c b/test/wycheproof-app/hmac_sha224_test.json.c similarity index 99% rename from test/wycheproof/hmac_sha224_test.json.c rename to test/wycheproof-app/hmac_sha224_test.json.c index c8724fcebef15e0ead8852ab4dbc3e02bed214b6..72cc647e4ddfa8ac8c768274076e1b7d106e388a 100644 --- a/test/wycheproof/hmac_sha224_test.json.c +++ b/test/wycheproof-app/hmac_sha224_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/hmac_sha256_test.json.c b/test/wycheproof-app/hmac_sha256_test.json.c similarity index 99% rename from test/wycheproof/hmac_sha256_test.json.c rename to test/wycheproof-app/hmac_sha256_test.json.c index 68c7406e6f6cbaab783b0d433e5d837e3a06e0c9..2ec3800269f03f81dfb5c9a2adda402110cae8d2 100644 --- a/test/wycheproof/hmac_sha256_test.json.c +++ b/test/wycheproof-app/hmac_sha256_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/hmac_sha384_test.json.c b/test/wycheproof-app/hmac_sha384_test.json.c similarity index 99% rename from test/wycheproof/hmac_sha384_test.json.c rename to test/wycheproof-app/hmac_sha384_test.json.c index 09bd57fc021e044f1aa87a9ff6537a9f41553753..7a650d4236079bd581a528b7cb16a41185ca543d 100644 --- a/test/wycheproof/hmac_sha384_test.json.c +++ b/test/wycheproof-app/hmac_sha384_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof/hmac_sha512_test.json.c b/test/wycheproof-app/hmac_sha512_test.json.c similarity index 99% rename from test/wycheproof/hmac_sha512_test.json.c rename to test/wycheproof-app/hmac_sha512_test.json.c index 4e7b79fb526e902b01508e1cb3c70e2a05a24b6d..ecfd924391a82b679ca290a6f5cdcc8c2ca1531d 100644 --- a/test/wycheproof/hmac_sha512_test.json.c +++ b/test/wycheproof-app/hmac_sha512_test.json.c @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/test/wycheproof-app/win_x64.mak b/test/wycheproof-app/win_x64.mak new file mode 100644 index 0000000000000000000000000000000000000000..fe031ebb6016bf92b8d57ba76c10261e8844c8a6 --- /dev/null +++ b/test/wycheproof-app/win_x64.mak @@ -0,0 +1,57 @@ +# +# Copyright (c) 2022-2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP = imb-wycheproof + +include ..\common\win_x64_common.mk + +OBJS = aes_gcm_test.json.obj aes_ccm_test.json.obj \ + chacha20_poly1305_test.json.obj \ + aes_cmac_test.json.obj gmac_test.json.obj \ + hmac_sha1_test.json.obj hmac_sha224_test.json.obj \ + hmac_sha256_test.json.obj hmac_sha384_test.json.obj \ + hmac_sha512_test.json.obj wycheproof.obj +LFLAGS = /out:$(APP).exe $(DLFLAGS) + +all: $(APP).exe + +$(APP).exe: $(OBJS) $(IPSECLIB) + $(LNK) $(LFLAGS) $(OBJS) $(IPSECLIB) + +tests.dep: $(OBJS) + @type *.obj.dep > $@ 2> nul + +.c.obj: + $(CC) /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +clean: + del /q $(OBJS) tests.dep *.obj.dep $(APP).exe + +!if exist(tests.dep) +!include tests.dep +!endif diff --git a/test/wycheproof/wycheproof.c b/test/wycheproof-app/wycheproof.c similarity index 93% rename from test/wycheproof/wycheproof.c rename to test/wycheproof-app/wycheproof.c index ebf1ad48699dfb48bce49b6cf0815173253f1a98..8cefdda3fd90b57a15860a4f7101fd299522dfff 100644 --- a/test/wycheproof/wycheproof.c +++ b/test/wycheproof-app/wycheproof.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2022, Intel Corporation + Copyright (c) 2022-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -67,22 +67,6 @@ static int process_job(IMB_MGR *p_mgr) return 1; } -static void -prep_iopad(const size_t scratch_size, void *scratch, - const size_t key_size, const void *key, - const int pattern) -{ - uint8_t *cb = (uint8_t *) scratch; - const uint8_t *kp = (const uint8_t *) key; - const size_t max_j = - (key_size > scratch_size) ? scratch_size : key_size; - size_t j; - - memset(scratch, pattern, scratch_size); - for (j = 0; j < max_j; j++) - cb[j] ^= kp[j]; -} - #define PUTS_ONCE(_s) { \ static int _ran_already = 0; \ \ @@ -388,16 +372,12 @@ static int test_hmac_sha1(IMB_MGR *p_mgr) const struct mac_test *v = hmac_sha1_test_json; DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); - uint8_t scratch[IMB_SHA1_BLOCK_SIZE]; - uint8_t key[IMB_SHA1_DIGEST_SIZE_IN_BYTES]; uint8_t tag[IMB_SHA1_DIGEST_SIZE_IN_BYTES]; while (IMB_FLUSH_JOB(p_mgr) != NULL) ; for ( ; v->msg != NULL; v++, run_vectors++) { - const void *key_ptr = NULL; - size_t key_size = 0; IMB_ASSERT((v->tagSize / 8) <= sizeof(tag)); @@ -435,22 +415,8 @@ static int test_hmac_sha1(IMB_MGR *p_mgr) else job->auth_tag_output_len_in_bytes = v->tagSize / 8; - /* prepare key */ - if ((v->keySize / 8) <= IMB_SHA1_BLOCK_SIZE) { - key_ptr = v->key; - key_size = v->keySize / 8; - } else { - IMB_SHA1(p_mgr, v->key, v->keySize / 8, key); - key_ptr = key; - key_size = IMB_SHA1_DIGEST_SIZE_IN_BYTES; - } - - /* compute IPAD and OPAD */ - prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x36); - IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x5c); - IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_opad); + imb_hmac_ipad_opad(p_mgr, IMB_AUTH_HMAC_SHA_1, + v->key, v->keySize / 8, hmac_ipad, hmac_opad); job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; @@ -472,17 +438,12 @@ static int test_hmac_sha224(IMB_MGR *p_mgr) const struct mac_test *v = hmac_sha224_test_json; DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); - uint8_t scratch[IMB_SHA_256_BLOCK_SIZE]; - uint8_t key[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; uint8_t tag[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; while (IMB_FLUSH_JOB(p_mgr) != NULL) ; for ( ; v->msg != NULL; v++, run_vectors++) { - const void *key_ptr = NULL; - size_t key_size = 0; - IMB_ASSERT((v->tagSize / 8) <= sizeof(tag)); /* tag too long */ @@ -512,21 +473,8 @@ static int test_hmac_sha224(IMB_MGR *p_mgr) job->auth_tag_output = tag; job->auth_tag_output_len_in_bytes = v->tagSize / 8; - /* prepare key */ - if ((v->keySize / 8) <= IMB_SHA_256_BLOCK_SIZE) { - key_ptr = v->key; - key_size = v->keySize / 8; - } else { - IMB_SHA224(p_mgr, v->key, v->keySize / 8, key); - key_ptr = key; - key_size = IMB_SHA224_DIGEST_SIZE_IN_BYTES; - } - /* compute IPAD and OPAD */ - prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x36); - IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x5c); - IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_opad); + imb_hmac_ipad_opad(p_mgr, IMB_AUTH_HMAC_SHA_224, + v->key, v->keySize / 8, hmac_ipad, hmac_opad); job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; @@ -548,17 +496,12 @@ static int test_hmac_sha256(IMB_MGR *p_mgr) const struct mac_test *v = hmac_sha256_test_json; DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); - uint8_t scratch[IMB_SHA_256_BLOCK_SIZE]; - uint8_t key[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; uint8_t tag[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; while (IMB_FLUSH_JOB(p_mgr) != NULL) ; for ( ; v->msg != NULL; v++, run_vectors++) { - const void *key_ptr = NULL; - size_t key_size = 0; - IMB_ASSERT((v->tagSize / 8) <= sizeof(tag)); /* tag too long */ @@ -588,22 +531,8 @@ static int test_hmac_sha256(IMB_MGR *p_mgr) job->auth_tag_output = tag; job->auth_tag_output_len_in_bytes = v->tagSize / 8; - /* prepare key */ - if ((v->keySize / 8) <= IMB_SHA_256_BLOCK_SIZE) { - key_ptr = v->key; - key_size = v->keySize / 8; - } else { - IMB_SHA256(p_mgr, v->key, v->keySize / 8, key); - key_ptr = key; - key_size = IMB_SHA256_DIGEST_SIZE_IN_BYTES; - } - - /* compute IPAD and OPAD */ - prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x36); - IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x5c); - IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_opad); + imb_hmac_ipad_opad(p_mgr, IMB_AUTH_HMAC_SHA_256, + v->key, v->keySize / 8, hmac_ipad, hmac_opad); job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; @@ -625,7 +554,7 @@ static int test_hmac_sha384(IMB_MGR *p_mgr) const struct mac_test *v = hmac_sha384_test_json; DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; + uint8_t scratch[IMB_SHA_384_BLOCK_SIZE]; while (IMB_FLUSH_JOB(p_mgr) != NULL) ; @@ -660,14 +589,8 @@ static int test_hmac_sha384(IMB_MGR *p_mgr) job->auth_tag_output = scratch; job->auth_tag_output_len_in_bytes = v->tagSize / 8; - /* compute IPAD and OPAD */ - prep_iopad(sizeof(scratch), scratch, - v->keySize / 8, v->key, 0x36); - IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - prep_iopad(sizeof(scratch), scratch, - v->keySize / 8, v->key, 0x5c); - IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_opad); + imb_hmac_ipad_opad(p_mgr, IMB_AUTH_HMAC_SHA_384, + v->key, v->keySize / 8, hmac_ipad, hmac_opad); job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; @@ -724,14 +647,8 @@ static int test_hmac_sha512(IMB_MGR *p_mgr) job->auth_tag_output = scratch; job->auth_tag_output_len_in_bytes = v->tagSize / 8; - /* compute IPAD and OPAD */ - prep_iopad(sizeof(scratch), scratch, - v->keySize / 8, v->key, 0x36); - IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_ipad); - - prep_iopad(sizeof(scratch), scratch, - v->keySize / 8, v->key, 0x5c); - IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_opad); + imb_hmac_ipad_opad(p_mgr, IMB_AUTH_HMAC_SHA_512, + v->key, v->keySize / 8, hmac_ipad, hmac_opad); job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; diff --git a/test/xcbc_test.c b/test/xcbc_test.c deleted file mode 100644 index 6204f0bf9574c4fe0f3e884d53b4e43cf54aa803..0000000000000000000000000000000000000000 --- a/test/xcbc_test.c +++ /dev/null @@ -1,541 +0,0 @@ -/***************************************************************************** - Copyright (c) 2020-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include -#include -#include - -#include -#include "gcm_ctr_vectors_test.h" -#include "utils.h" - - -int xcbc_test(struct IMB_MGR *mb_mgr); - -/* - * Test vectors from https://tools.ietf.org/html/rfc3566 - */ - -static const uint8_t key[16] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f -}; - -/** - * Test Case #1 : AES-XCBC-MAC-96 with 0-byte input - * Key (K) : 000102030405060708090a0b0c0d0e0f - * Message (M) : - * AES-XCBC-MAC : 75f0251d528ac01c4573dfd584d79f29 - * AES-XCBC-MAC-96: 75f0251d528ac01c4573dfd5 - */ -static const uint8_t M_1[] = {0}; - -static const uint8_t T_1[12] = { - 0x75, 0xf0, 0x25, 0x1d, 0x52, 0x8a, 0xc0, 0x1c, - 0x45, 0x73, 0xdf, 0xd5 -}; - -/** - * Test Case #2 : AES-XCBC-MAC-96 with 3-byte input - * Key (K) : 000102030405060708090a0b0c0d0e0f - * Message (M) : 000102 - * AES-XCBC-MAC : 5b376580ae2f19afe7219ceef172756f - * AES-XCBC-MAC-96: 5b376580ae2f19afe7219cee - */ -static const uint8_t M_2[3] = { - 0x00, 0x01, 0x02 -}; -static const uint8_t T_2[12] = { - 0x5b, 0x37, 0x65, 0x80, 0xae, 0x2f, 0x19, 0xaf, - 0xe7, 0x21, 0x9c, 0xee -}; - -/** - * Test Case #3 : AES-XCBC-MAC-96 with 16-byte input - * Key (K) : 000102030405060708090a0b0c0d0e0f - * Message (M) : 000102030405060708090a0b0c0d0e0f - * AES-XCBC-MAC : d2a246fa349b68a79998a4394ff7a263 - * AES-XCBC-MAC-96: d2a246fa349b68a79998a439 - */ -static const uint8_t M_3[16] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f -}; -static const uint8_t T_3[12] = { - 0xd2, 0xa2, 0x46, 0xfa, 0x34, 0x9b, 0x68, 0xa7, - 0x99, 0x98, 0xa4, 0x39 -}; - -/** - * Test Case #4 : AES-XCBC-MAC-96 with 20-byte input - * Key (K) : 000102030405060708090a0b0c0d0e0f - * Message (M) : 000102030405060708090a0b0c0d0e0f10111213 - * AES-XCBC-MAC : 47f51b4564966215b8985c63055ed308 - * AES-XCBC-MAC-96: 47f51b4564966215b8985c63 - */ -static const uint8_t M_4[20] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13 -}; -static const uint8_t T_4[12] = { - 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15, - 0xb8, 0x98, 0x5c, 0x63 -}; - -/** - * Test Case #5 : AES-XCBC-MAC-96 with 32-byte input - * Key (K) : 000102030405060708090a0b0c0d0e0f - * Message (M) : 000102030405060708090a0b0c0d0e0f10111213141516171819 - * 1a1b1c1d1e1f - * AES-XCBC-MAC : f54f0ec8d2b9f3d36807734bd5283fd4 - * AES-XCBC-MAC-96: f54f0ec8d2b9f3d36807734b - */ -static const uint8_t M_5[32] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f -}; -static const uint8_t T_5[12] = { - 0xf5, 0x4f, 0x0e, 0xc8, 0xd2, 0xb9, 0xf3, 0xd3, - 0x68, 0x07, 0x73, 0x4b -}; - -/** - * Test Case #6 : AES-XCBC-MAC-96 with 34-byte input - * Key (K) : 000102030405060708090a0b0c0d0e0f - * Message (M) : 000102030405060708090a0b0c0d0e0f10111213141516171819 - * 1a1b1c1d1e1f2021 - * AES-XCBC-MAC : becbb3bccdb518a30677d5481fb6b4d8 - * AES-XCBC-MAC-96: becbb3bccdb518a30677d548 - */ -static const uint8_t M_6[34] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21 -}; -static const uint8_t T_6[12] = { - 0xbe, 0xcb, 0xb3, 0xbc, 0xcd, 0xb5, 0x18, 0xa3, - 0x06, 0x77, 0xd5, 0x48 -}; - -/** - * Test Case #7 : AES-XCBC-MAC-96 with 1000-byte input - * Key (K) : 000102030405060708090a0b0c0d0e0f - * Message (M) : 00000000000000000000 ... 00000000000000000000 - * [1000 bytes] - * AES-XCBC-MAC : f0dafee895db30253761103b5d84528f - * AES-XCBC-MAC-96: f0dafee895db30253761103b - */ -static const uint8_t M_7[1000] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; -static const uint8_t T_7[12] = { - 0xf0, 0xda, 0xfe, 0xe8, 0x95, 0xdb, 0x30, 0x25, - 0x37, 0x61, 0x10, 0x3b -}; - -static const struct xcbc_rfc3566_vector { - const uint8_t *key; - const uint8_t *M; - size_t len; - const uint8_t *T; - size_t T_len; -} xcbc_vectors[] = { - { key, M_1, 0, T_1, 12 }, - { key, M_2, 3, T_2, 12 }, - { key, M_3, 16, T_3, 12 }, - { key, M_4, 20, T_4, 12 }, - { key, M_5, 32, T_5, 12 }, - { key, M_6, 34, T_6, 12 }, - { key, M_7, 1000, T_7, 12 }, -}; - -static int -xcbc_job_ok(const struct xcbc_rfc3566_vector *vec, - const struct IMB_JOB *job, - const uint8_t *auth, - const uint8_t *padding, - const size_t sizeof_padding) -{ - const size_t auth_len = job->auth_tag_output_len_in_bytes; - - if (job->status != IMB_STATUS_COMPLETED) { - printf("%d Error status:%d", __LINE__, job->status); - return 0; - } - - /* hash checks */ - if (memcmp(padding, &auth[sizeof_padding + auth_len], - sizeof_padding)) { - printf("hash overwrite tail\n"); - hexdump(stderr, "Target", - &auth[sizeof_padding + auth_len], sizeof_padding); - return 0; - } - - if (memcmp(padding, &auth[0], sizeof_padding)) { - printf("hash overwrite head\n"); - hexdump(stderr, "Target", &auth[0], sizeof_padding); - return 0; - } - - if (memcmp(vec->T, &auth[sizeof_padding], auth_len)) { - printf("hash mismatched\n"); - hexdump(stderr, "Received", &auth[sizeof_padding], - auth_len); - hexdump(stderr, "Expected", vec->T, - auth_len); - return 0; - } - return 1; -} - -static int -test_xcbc(struct IMB_MGR *mb_mgr, - const struct xcbc_rfc3566_vector *vec, - const int dir, - const int num_jobs) -{ - DECLARE_ALIGNED(uint32_t k1_exp[4*11], 16); - uint8_t k2[16], k3[16]; - struct IMB_JOB *job; - uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); - int i = 0, jobs_rx = 0, ret = -1; - - if (auths == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end2; - } - - memset(padding, -1, sizeof(padding)); - memset(auths, 0, num_jobs * sizeof(void *)); - - for (i = 0; i < num_jobs; i++) { - auths[i] = malloc(16 + (sizeof(padding) * 2)); - if (auths[i] == NULL) { - fprintf(stderr, "Can't allocate buffer memory\n"); - goto end; - } - - memset(auths[i], -1, 16 + (sizeof(padding) * 2)); - } - - IMB_AES_XCBC_KEYEXP(mb_mgr, vec->key, k1_exp, k2, k3); - - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - /** - * Submit all jobs then flush any outstanding jobs - */ - for (i = 0; i < num_jobs; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - job->cipher_direction = dir; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_AES_XCBC; - job->msg_len_to_hash_in_bytes = vec->len; - job->u.XCBC._k1_expanded = k1_exp; - job->u.XCBC._k2 = k2; - job->u.XCBC._k3 = k3; - job->src = vec->M; - job->hash_start_src_offset_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->T_len; - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job) { - jobs_rx++; - if (num_jobs < 4) { - printf("%d Unexpected return from submit_job\n", - __LINE__); - goto end; - } - if (!xcbc_job_ok(vec, job, job->user_data, padding, - sizeof(padding))) - goto end; - } - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - jobs_rx++; - - if (!xcbc_job_ok(vec, job, job->user_data, padding, - sizeof(padding))) - goto end; - } - - if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); - goto end; - } - - /** - * Submit each job and flush immediately - */ - for (i = 0; i < num_jobs; i++) { - struct IMB_JOB *first_job = NULL; - - job = IMB_GET_NEXT_JOB(mb_mgr); - first_job = job; - - job->cipher_direction = dir; - job->chain_order = IMB_ORDER_HASH_CIPHER; - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = IMB_AUTH_AES_XCBC; - job->msg_len_to_hash_in_bytes = vec->len; - job->u.XCBC._k1_expanded = k1_exp; - job->u.XCBC._k2 = k2; - job->u.XCBC._k3 = k3; - job->src = vec->M; - job->hash_start_src_offset_in_bytes = 0; - job->auth_tag_output = auths[i] + sizeof(padding); - job->auth_tag_output_len_in_bytes = vec->T_len; - - job->user_data = auths[i]; - - job = IMB_SUBMIT_JOB(mb_mgr); - if (job != NULL) { - printf("Received job, expected NULL\n"); - goto end; - } - - while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { - if (job != first_job) { - printf("Invalid return job received\n"); - goto end; - } - if (!xcbc_job_ok(vec, job, job->user_data, padding, - sizeof(padding))) - goto end; - } - } - - ret = 0; - - end: - while (IMB_FLUSH_JOB(mb_mgr) != NULL) - ; - - for (i = 0; i < num_jobs; i++) { - if (auths[i] != NULL) - free(auths[i]); - } - - end2: - if (auths != NULL) - free(auths); - - return ret; -} - -static void -test_xcbc_std_vectors(struct IMB_MGR *mb_mgr, - struct test_suite_context *ctx, - const int num_jobs) -{ - const int vectors_cnt = DIM(xcbc_vectors); - int vect; - - printf("AES-XCBC-128 standard test vectors (N jobs = %d):\n", num_jobs); - for (vect = 1; vect <= vectors_cnt; vect++) { - const int idx = vect - 1; -#ifdef DEBUG - printf("Standard XCBC-128 vector [%d/%d] M len: %d, T len:%d\n", - vect, vectors_cnt, - (int) xcbc_vectors[idx].len, - (int) xcbc_vectors[idx].T_len); -#else - printf("."); -#endif - - if (test_xcbc(mb_mgr, &xcbc_vectors[idx], - IMB_DIR_ENCRYPT, num_jobs)) { - printf("error #%d encrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - if (test_xcbc(mb_mgr, &xcbc_vectors[idx], - IMB_DIR_DECRYPT, num_jobs)) { - printf("error #%d decrypt\n", vect); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - } - printf("\n"); -} - -int -xcbc_test(struct IMB_MGR *mb_mgr) -{ - struct test_suite_context ctx; - int i, errors; - - test_suite_start(&ctx, "AES-XCBC-128"); - /* AES-XCBC 128 with standard vectors */ - for (i = 1; i < 20; i++) - test_xcbc_std_vectors(mb_mgr, &ctx, i); - errors = test_suite_end(&ctx); - - return errors; -} diff --git a/test/xvalid-app/CMakeLists.txt b/test/xvalid-app/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..f416ab324d33eb06c0500064e245c10ec5e183bb --- /dev/null +++ b/test/xvalid-app/CMakeLists.txt @@ -0,0 +1,105 @@ +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +######################################## +# set app and library names +######################################## +set(XVALID_APP imb-xvalid) +set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib) +set(TEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include) + +######################################## +# set imb-xvalid source files +######################################## +set(XVALID_APP_SRC_FILES_C + ${CMAKE_CURRENT_SOURCE_DIR}/ipsec_xvalid.c + ${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.c +) +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(XVALID_APP_SRC_FILES_ASM + ${CMAKE_CURRENT_SOURCE_DIR}/misc_aarch64.S + ) +else() + set(XVALID_APP_SRC_FILES_ASM + ${CMAKE_CURRENT_SOURCE_DIR}/misc.asm + ) +endif() + +######################################## +# setup NASM +######################################## +if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + enable_language(ASM) + if(NOT CMAKE_ASM_COMPILER_LOADED) + message(FATAL_ERROR "Can't find assembler") + endif() +else() + enable_language(ASM_NASM) + if(NOT CMAKE_ASM_NASM_COMPILER_LOADED) + message(FATAL_ERROR "Can't find assembler") + endif() +endif() +set(CAN_USE_ASSEMBLER TRUE) + +######################################## +# set C compiler and NASM options +######################################## +# add OS specific options +if(WINDOWS) + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/windows.cmake) +else() + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/unix.cmake) +endif() + + +######################################## +# add targets +######################################## +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${LIB_DIR} ${TEST_INCLUDE_DIR}) +link_directories(${LIB_DIR}) + +add_executable(${XVALID_APP} ${XVALID_APP_SRC_FILES_C} ${XVALID_APP_SRC_FILES_ASM}) +target_link_libraries(${XVALID_APP} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${XVALID_APP} PRIVATE ${APP_DEFINES}) + +######################################## +# add tests +######################################## + +add_test(NAME XVALID-64B + COMMAND ${XVALID_APP} --job-size 64) + +# run safe check only when SAFE_DATA is enabled and only run on release build +if (SAFE_DATA) + add_test(NAME XVALID-64B-SAFE-CHECK + COMMAND ${XVALID_APP} --safe-check --job-size 64 + CONFIGURATIONS Release) +endif() + diff --git a/test/xvalid-app/Makefile b/test/xvalid-app/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..3d947a21541088082e704f0904e7410fa90a745e --- /dev/null +++ b/test/xvalid-app/Makefile @@ -0,0 +1,77 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP := imb-xvalid + +# include common options +include ../common/common.mk + +# imb-xvalid modules +ifeq ($(ARCH),x86_64) +ASM := misc.asm +SOURCES := ipsec_xvalid.c +OBJECTS := $(SOURCES:%.c=%.o) $(ASM:%.asm=%.o) utils.o + +# rule for compiling assembly code with producing dependencies +%.o:%.asm +ifeq ($(USE_YASM),y) + $(YASM) $(YASM_FLAGS) $< -o $@ +else + $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< +endif +ifeq ($(CC_HAS_CET),1) + $(LD) $(CET_LDFLAGS) -o $@.tmp $@ + mv $@.tmp $@ +endif + +else # x86_64 +ASM := misc_aarch64.S +SOURCES := ipsec_xvalid.c +OBJECTS := $(SOURCES:%.c=%.o) $(ASM:%.S=%.o) utils.o + +# rule for compiling assembly code with producing dependencies +%.o:%.S + $(CC) -c $(CFLAGS) $< -o $@ +endif # x86_64 + +# targets come here +all: $(APP) + +$(APP): $(OBJECTS) + $(CC) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +utils.o: ../common/utils.c + $(CC) -c $(CFLAGS) $< -o $@ + +.PHONY: clean +clean: + -rm -f $(DEP_FILES) $(OBJECTS) $(APP) $(ASM_OBJECTS) + +# if target not clean then make dependencies +ifneq ($(MAKECMDGOALS),clean) +-include $(wildcard *.d) +endif diff --git a/test/xvalid-app/README.md b/test/xvalid-app/README.md new file mode 100644 index 0000000000000000000000000000000000000000..644e913dde4e67af8e979f09ebcb05ac1342a325 --- /dev/null +++ b/test/xvalid-app/README.md @@ -0,0 +1,62 @@ +# Intel(R) Multi-Buffer Crypto for IPsec Library - Cross Validation Application + +## Contents + +- Overview +- Usage + + +## Overview + +The cross validation application validates correct algorithm implementation by encrypting +randomly generated data with one architectural implementation and decrypting with another. +The decrypted data is verified by comparing against the original. By default, the +tool will cross validate all algorithms across all combinations of architectures. + +For release builds of the library with the SAFE_DATA option enabled (default setting), +the cross validation application can be used to check all sensitive data has been cleared +from processor registers and memory. + + +## Usage + +Before running the application, ensure the library is installed by following the instructions +in the [README](https://github.com/intel/intel-ipsec-mb/tree/main/test#library-installation). + +### Linux +To cross validate all algorithms across all architectures: +`./imb-xvalid` + +To validate all SSE algorithm implementations against AVX512: +`./imb-xvalid --enc-arch SSE --dec-arch AVX512` + +To validate AES-CBC-128 AVX algorithm implementation against SSE: +`./imb-xvalid --enc-arch AVX --dec-arch SSE --cipher-algo aes-cbc-128` + +To validate AES-GCM-128 using only 512 byte buffers: +`./imb-xvalid --aead-algo aes-gcm-128 --job-size 512` + +To check for sensitive data left in registers or memory: +`./imb-xvalid --safe-check` + +To display an extensive help page: +`./imb-xvalid --help` + +### Windows +To cross validate all algorithms across all architectures: +`imb-xvalid.exe` + +To validate all SSE algorithm implementations against AVX512: +`imb-xvalid.exe --enc-arch SSE --dec-arch AVX512` + +To validate AES-CBC-128 AVX algorithm implementation against SSE: +`imb-xvalid.exe --enc-arch AVX --dec-arch SSE --cipher-algo aes-cbc-128` + +To validate AES-GCM-128 using only 512 byte buffers: +`imb-xvalid.exe --aead-algo aes-gcm-128 --job-size 512` + +To check for sensitive data left in registers or memory: +`imb-xvalid.exe --safe-check` + +To display an extensive help page: +`imb-xvalid.exe --help` diff --git a/test/ipsec_xvalid.c b/test/xvalid-app/ipsec_xvalid.c similarity index 94% rename from test/ipsec_xvalid.c rename to test/xvalid-app/ipsec_xvalid.c index 8fdd557db4c4d4c75df9e832bdaf82a4960870b2..dcc7d8dd3b23b3a14832e4ffa537329bc1cab23c 100644 --- a/test/ipsec_xvalid.c +++ b/test/xvalid-app/ipsec_xvalid.c @@ -1,5 +1,5 @@ /********************************************************************** - Copyright(c) 2019-2022, Intel Corporation All rights reserved. + Copyright(c) 2019-2023, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -50,15 +50,14 @@ #define __func__ __FUNCTION__ #define strcasecmp _stricmp #else +#ifndef __aarch64__ +#include +#endif #define BSWAP64 __builtin_bswap64 #endif #include -#ifdef __x86_64__ -#include -#endif - /* maximum size of a test buffer */ #define JOB_SIZE_TOP (16 * 1024) @@ -106,10 +105,10 @@ static uint64_t pattern8_plain_text; struct params_s { IMB_CIPHER_MODE cipher_mode; /* CBC, CNTR, DES, GCM etc. */ IMB_HASH_ALG hash_alg; /* SHA-1 or others... */ - uint32_t key_size; - uint32_t buf_size; - uint64_t aad_size; - uint32_t num_sizes; + uint32_t key_size; + uint32_t buf_size; + uint64_t aad_size; + uint32_t num_sizes; }; /* Struct storing all expanded keys */ @@ -119,8 +118,8 @@ struct cipher_auth_keys { uint8_t ipad[IMB_SHA512_DIGEST_SIZE_IN_BYTES]; uint8_t opad[IMB_SHA512_DIGEST_SIZE_IN_BYTES]; DECLARE_ALIGNED(uint32_t k1_expanded[15 * 4], 16); - DECLARE_ALIGNED(uint8_t k2[32], 16); - DECLARE_ALIGNED(uint8_t k3[16], 16); + DECLARE_ALIGNED(uint8_t k2[32], 16); + DECLARE_ALIGNED(uint8_t k3[16], 16); DECLARE_ALIGNED(uint32_t enc_keys[15 * 4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15 * 4], 16); DECLARE_ALIGNED(struct gcm_key_data gdata_key, 64); @@ -159,17 +158,23 @@ struct str_value_mapping { }; const struct str_value_mapping arch_str_map[] = { +#ifndef __aarch64__ {.name = "NONE", .values.arch_type = IMB_ARCH_NONE }, - {.name = "NO-AESNI", .values.arch_type = IMB_ARCH_NOAESNI }, {.name = "SSE", .values.arch_type = IMB_ARCH_SSE }, + {.name = "NO-AESNI", .values.arch_type = IMB_ARCH_NOAESNI }, {.name = "AVX", .values.arch_type = IMB_ARCH_AVX }, {.name = "AVX2", .values.arch_type = IMB_ARCH_AVX2 }, - {.name = "AVX512", .values.arch_type = IMB_ARCH_AVX512 }, + {.name = "AVX512", .values.arch_type = IMB_ARCH_AVX512 } +#else + {.name = "NONE", .values.arch_type = IMB_ARCH_NONE }, + {.name = "NO-AESNI", .values.arch_type = IMB_ARCH_NOAESNI }, {.name = "AARCH64", .values.arch_type = IMB_ARCH_AARCH64 }, - {.name = "SVE256", .values.arch_type = IMB_ARCH_SVE256 }, + {.name = "SVE256", .values.arch_type = IMB_ARCH_SVE256 } +#endif }; struct str_value_mapping cipher_algo_str_map[] = { +#ifndef __aarch64__ { .name = "AES-CBC-128", .values.job_params = { @@ -289,6 +294,7 @@ struct str_value_mapping cipher_algo_str_map[] = { .key_size = 24 } }, +#endif /* __aarch64__ */ { .name = "ZUC-EEA3", .values.job_params = { @@ -310,6 +316,7 @@ struct str_value_mapping cipher_algo_str_map[] = { .key_size = 16 } }, +#ifndef __aarch64__ { .name = "KASUMI-F8", .values.job_params = { @@ -338,6 +345,7 @@ struct str_value_mapping cipher_algo_str_map[] = { .key_size = 32 } }, +#endif /* __aarch64__ */ { .name = "NULL-CIPHER", .values.job_params = { @@ -348,6 +356,7 @@ struct str_value_mapping cipher_algo_str_map[] = { }; struct str_value_mapping hash_algo_str_map[] = { +#ifndef __aarch64__ { .name = "HMAC-SHA1", .values.job_params = { @@ -396,12 +405,14 @@ struct str_value_mapping hash_algo_str_map[] = { .hash_alg = IMB_AUTH_AES_CMAC } }, +#endif /* __aarch64__ */ { .name = "NULL-HASH", .values.job_params = { .hash_alg = IMB_AUTH_NULL } }, +#ifndef __aarch64__ { .name = "AES-CMAC-128-BIT-LENGTH", .values.job_params = { @@ -438,6 +449,7 @@ struct str_value_mapping hash_algo_str_map[] = { .hash_alg = IMB_AUTH_SHA_512 } }, +#endif /* __aarch64__ */ { .name = "ZUC-EIA3", .values.job_params = { @@ -450,6 +462,7 @@ struct str_value_mapping hash_algo_str_map[] = { .hash_alg = IMB_AUTH_SNOW3G_UIA2_BITLEN, } }, +#ifndef __aarch64__ { .name = "KASUMI-F9", .values.job_params = { @@ -492,21 +505,25 @@ struct str_value_mapping hash_algo_str_map[] = { .hash_alg = IMB_AUTH_POLY1305, } }, +#endif /* __aarch64__ */ { .name = "ZUC-EIA3-256", .values.job_params = { .hash_alg = IMB_AUTH_ZUC256_EIA3_BITLEN, } }, +#ifndef __aarch64__ { .name = "GHASH", .values.job_params = { .hash_alg = IMB_AUTH_GHASH, } }, +#endif /* __aarch64__ */ }; struct str_value_mapping aead_algo_str_map[] = { +#ifndef __aarch64__ { .name = "AES-GCM-128", .values.job_params = { @@ -579,6 +596,7 @@ struct str_value_mapping aead_algo_str_map[] = { .key_size = 32 } }, +#endif /* __aarch64__ */ }; /* This struct stores all information about performed test case */ @@ -691,15 +709,20 @@ struct custom_job_params custom_job_params = { }; /* AESNI_EMU disabled by default */ -uint8_t enc_archs[IMB_ARCH_NUM] = {0, 0, 1, 1, 1, 1, 1, 1}; -uint8_t dec_archs[IMB_ARCH_NUM] = {0, 0, 1, 1, 1, 1, 1, 1}; +#ifndef __aarch64__ +uint8_t enc_archs[IMB_ARCH_NUM] = {0, 0, 1, 1, 1, 1}; +uint8_t dec_archs[IMB_ARCH_NUM] = {0, 0, 1, 1, 1, 1}; +#else +uint8_t enc_archs[IMB_ARCH_NUM] = {0, 0, 1, 1}; +uint8_t dec_archs[IMB_ARCH_NUM] = {0, 0, 1, 1}; +#endif uint64_t flags = 0; /* flags passed to alloc_mb_mgr() */ /* 0 => not possible, 1 => possible */ int is_avx_sse_check_possible = 0; -#ifdef __x86_64__ +#ifndef __aarch64__ static void avx_sse_check(const char *ctx_str, const IMB_HASH_ALG hash_alg, @@ -763,11 +786,11 @@ static void generate_patterns(void) pattern_cipher_key == 0 || pattern_plain_text == 0); - NOSIMD_MEMSET(&pattern8_auth_key, pattern_auth_key, + nosimd_memset(&pattern8_auth_key, pattern_auth_key, sizeof(pattern8_auth_key)); - NOSIMD_MEMSET(&pattern8_cipher_key, pattern_cipher_key, + nosimd_memset(&pattern8_cipher_key, pattern_cipher_key, sizeof(pattern8_cipher_key)); - NOSIMD_MEMSET(&pattern8_plain_text, pattern_plain_text, + nosimd_memset(&pattern8_plain_text, pattern_plain_text, sizeof(pattern8_plain_text)); printf(">>> Patterns: AUTH_KEY = 0x%02x, CIPHER_KEY = 0x%02x, " @@ -818,7 +841,7 @@ search_patterns(const void *ptr, const size_t mem_size) const size_t len_to_print = (mem_size - i) > sizeof(tb) ? sizeof(tb) : mem_size - i; - NOSIMD_MEMCPY(tb, &ptr8[i], len_to_print); + nosimd_memcpy(tb, &ptr8[i], len_to_print); fprintf(stderr, "%s\n", err_str); fprintf(stderr, "Offset = %zu bytes, Addr = %p, RSP = %p\n", @@ -874,6 +897,7 @@ print_algo_info(const struct params_s *params) struct custom_job_params *job_params; uint32_t i; +#ifndef __aarch64__ for (i = 0; i < DIM(aead_algo_str_map); i++) { job_params = &aead_algo_str_map[i].values.job_params; if (job_params->cipher_mode == params->cipher_mode && @@ -883,6 +907,7 @@ print_algo_info(const struct params_s *params) return; } } +#endif /* __aarch64__ */ for (i = 0; i < DIM(cipher_algo_str_map); i++) { job_params = &cipher_algo_str_map[i].values.job_params; @@ -908,7 +933,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, const uint32_t buf_size, const uint8_t tag_size, IMB_CIPHER_DIRECTION cipher_dir, struct cipher_auth_keys *keys, uint8_t *cipher_iv, - uint8_t *auth_iv, unsigned index, uint8_t *next_iv) + uint8_t *auth_iv, const unsigned index, uint8_t *next_iv) { static const void *ks_ptr[3]; uint32_t *k1_expanded = keys->k1_expanded; @@ -970,6 +995,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->hash_alg = params->hash_alg; switch (params->hash_alg) { +#ifndef __aarch64__ case IMB_AUTH_AES_XCBC: job->u.XCBC._k1_expanded = k1_expanded; job->u.XCBC._k2 = k2; @@ -1009,6 +1035,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->u.HMAC._hashed_auth_key_xor_opad = (uint8_t *) opad; break; +#endif /* __aarch64__ */ case IMB_AUTH_ZUC256_EIA3_BITLEN: job->u.ZUC_EIA3._key = k2; if (auth_iv_size == 23) { @@ -1033,6 +1060,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->msg_len_to_hash_in_bits = (job->msg_len_to_hash_in_bytes * 8); break; +#ifndef __aarch64__ case IMB_AUTH_KASUMI_UIA1: job->u.KASUMI_UIA1._key = k2; break; @@ -1048,7 +1076,9 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->u.GHASH._init_tag = auth_iv; break; case IMB_AUTH_PON_CRC_BIP: +#endif /* __aarch64__ */ case IMB_AUTH_NULL: +#ifndef __aarch64__ case IMB_AUTH_AES_GMAC: case IMB_AUTH_AES_CCM: case IMB_AUTH_SHA_1: @@ -1084,6 +1114,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, case IMB_AUTH_SNOW_V_AEAD: job->u.SNOW_V_AEAD.aad_len_in_bytes = params->aad_size; job->u.SNOW_V_AEAD.aad = aad; +#endif /* __aarch64__ */ break; default: printf("Unsupported hash algorithm %u, line %d\n", @@ -1116,6 +1147,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->key_len_in_bytes = params->key_size; switch (job->cipher_mode) { +#ifndef __aarch64__ case IMB_CIPHER_CBC: case IMB_CIPHER_DOCSIS_SEC_BPI: case IMB_CIPHER_CBCS_1_9: @@ -1166,6 +1198,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->dec_keys = dec_keys; job->iv_len_in_bytes = 0; break; +#endif /* __aarch64__ */ case IMB_CIPHER_ZUC_EEA3: job->enc_keys = k2; job->dec_keys = k2; @@ -1182,6 +1215,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->msg_len_to_cipher_in_bits = (job->msg_len_to_cipher_in_bytes * 8); break; +#ifndef __aarch64__ case IMB_CIPHER_KASUMI_UEA1_BITLEN: job->enc_keys = k2; job->dec_keys = k2; @@ -1203,6 +1237,7 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->dec_keys = k2; job->iv_len_in_bytes = 16; break; +#endif /* __aarch64__ */ case IMB_CIPHER_NULL: /* No operation needed */ break; @@ -1227,7 +1262,6 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, const struct params_s *params, const unsigned int force_pattern) { - uint8_t *buf = keys->temp_buf; uint32_t *dust = keys->dust; uint32_t *k1_expanded = keys->k1_expanded; uint8_t *k2 = keys->k2; @@ -1237,30 +1271,27 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, uint8_t *ipad = keys->ipad; uint8_t *opad = keys->opad; struct gcm_key_data *gdata_key = &keys->gdata_key; - uint8_t i; /* Set all expanded keys to pattern_cipher_key/pattern_auth_key * if flag is set */ if (force_pattern) { switch (params->hash_alg) { +#ifndef __aarch64__ case IMB_AUTH_AES_XCBC: - NOSIMD_MEMSET(k1_expanded, pattern_auth_key, + nosimd_memset(k1_expanded, pattern_auth_key, sizeof(keys->k1_expanded)); - break; case IMB_AUTH_AES_CMAC: case IMB_AUTH_AES_CMAC_BITLEN: case IMB_AUTH_AES_CMAC_256: - NOSIMD_MEMSET(k1_expanded, pattern_auth_key, + nosimd_memset(k1_expanded, pattern_auth_key, sizeof(keys->k1_expanded)); - NOSIMD_MEMSET(k2, pattern_auth_key, sizeof(keys->k2)); - NOSIMD_MEMSET(k3, pattern_auth_key, sizeof(keys->k3)); - + nosimd_memset(k2, pattern_auth_key, sizeof(keys->k2)); + nosimd_memset(k3, pattern_auth_key, sizeof(keys->k3)); break; case IMB_AUTH_POLY1305: - NOSIMD_MEMSET(k1_expanded, pattern_auth_key, + nosimd_memset(k1_expanded, pattern_auth_key, sizeof(keys->k1_expanded)); - break; case IMB_AUTH_HMAC_SHA_1: case IMB_AUTH_HMAC_SHA_224: @@ -1268,19 +1299,21 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_AUTH_HMAC_SHA_384: case IMB_AUTH_HMAC_SHA_512: case IMB_AUTH_MD5: - NOSIMD_MEMSET(ipad, pattern_auth_key, + nosimd_memset(ipad, pattern_auth_key, sizeof(keys->ipad)); - NOSIMD_MEMSET(opad, pattern_auth_key, + nosimd_memset(opad, pattern_auth_key, sizeof(keys->opad)); - break; +#endif /* __aarch64__ */ case IMB_AUTH_ZUC_EIA3_BITLEN: case IMB_AUTH_ZUC256_EIA3_BITLEN: case IMB_AUTH_SNOW3G_UIA2_BITLEN: +#ifndef __aarch64__ case IMB_AUTH_KASUMI_UIA1: - NOSIMD_MEMSET(k3, pattern_auth_key, sizeof(keys->k3)); - +#endif /* __aarch64__ */ + nosimd_memset(k3, pattern_auth_key, sizeof(keys->k3)); break; +#ifndef __aarch64__ case IMB_AUTH_AES_CCM: case IMB_AUTH_AES_GMAC: case IMB_AUTH_NULL: @@ -1313,9 +1346,14 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_AUTH_AES_GMAC_192: case IMB_AUTH_AES_GMAC_256: case IMB_AUTH_GHASH: - NOSIMD_MEMSET(gdata_key, pattern_auth_key, + nosimd_memset(gdata_key, pattern_auth_key, sizeof(keys->gdata_key)); break; +#else /* __aarch64__ */ + case IMB_AUTH_NULL: + /* No operation needed */ + break; +#endif /* __aarch64__ */ default: fprintf(stderr, "Unsupported hash algorithm %u, line %d\n", @@ -1324,8 +1362,9 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, } switch (params->cipher_mode) { +#ifndef __aarch64__ case IMB_CIPHER_GCM: - NOSIMD_MEMSET(gdata_key, pattern_cipher_key, + nosimd_memset(gdata_key, pattern_cipher_key, sizeof(keys->gdata_key)); break; case IMB_CIPHER_PON_AES_CNTR: @@ -1336,28 +1375,33 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_CIPHER_DOCSIS_SEC_BPI: case IMB_CIPHER_ECB: case IMB_CIPHER_CBCS_1_9: - NOSIMD_MEMSET(enc_keys, pattern_cipher_key, + nosimd_memset(enc_keys, pattern_cipher_key, sizeof(keys->enc_keys)); - NOSIMD_MEMSET(dec_keys, pattern_cipher_key, + nosimd_memset(dec_keys, pattern_cipher_key, sizeof(keys->dec_keys)); break; case IMB_CIPHER_DES: case IMB_CIPHER_DES3: case IMB_CIPHER_DOCSIS_DES: - NOSIMD_MEMSET(enc_keys, pattern_cipher_key, + nosimd_memset(enc_keys, pattern_cipher_key, sizeof(keys->enc_keys)); break; +#endif /* __aarch64__ */ case IMB_CIPHER_SNOW3G_UEA2_BITLEN: +#ifndef __aarch64__ case IMB_CIPHER_KASUMI_UEA1_BITLEN: - NOSIMD_MEMSET(k2, pattern_cipher_key, 16); +#endif /* __aarch64__ */ + nosimd_memset(k2, pattern_cipher_key, 16); break; case IMB_CIPHER_ZUC_EEA3: +#ifndef __aarch64__ case IMB_CIPHER_CHACHA20: case IMB_CIPHER_CHACHA20_POLY1305: case IMB_CIPHER_CHACHA20_POLY1305_SGL: case IMB_CIPHER_SNOW_V: case IMB_CIPHER_SNOW_V_AEAD: - NOSIMD_MEMSET(k2, pattern_cipher_key, 32); +#endif /* __aarch64__ */ + nosimd_memset(k2, pattern_cipher_key, 32); break; case IMB_CIPHER_NULL: /* No operation needed */ @@ -1371,6 +1415,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, } switch (params->hash_alg) { +#ifndef __aarch64__ case IMB_AUTH_AES_XCBC: IMB_AES_XCBC_KEYEXP(mb_mgr, auth_key, k1_expanded, k2, k3); break; @@ -1384,99 +1429,24 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, IMB_AES_CMAC_SUBKEY_GEN_256(mb_mgr, k1_expanded, k2, k3); break; case IMB_AUTH_HMAC_SHA_1: - /* compute ipad hash */ - NOSIMD_MEMSET(buf, 0x36, IMB_SHA1_BLOCK_SIZE); - - for (i = 0; i < IMB_SHA1_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad); - - /* compute opad hash */ - NOSIMD_MEMSET(buf, 0x5c, IMB_SHA1_BLOCK_SIZE); - - for (i = 0; i < IMB_SHA1_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad); - - break; case IMB_AUTH_HMAC_SHA_224: - /* compute ipad hash */ - NOSIMD_MEMSET(buf, 0x36, IMB_SHA_256_BLOCK_SIZE); - - for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad); - - /* compute opad hash */ - NOSIMD_MEMSET(buf, 0x5c, IMB_SHA_256_BLOCK_SIZE); - - for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad); - - break; case IMB_AUTH_HMAC_SHA_256: - /* compute ipad hash */ - NOSIMD_MEMSET(buf, 0x36, IMB_SHA_256_BLOCK_SIZE); - for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad); - - /* compute opad hash */ - NOSIMD_MEMSET(buf, 0x5c, IMB_SHA_256_BLOCK_SIZE); - for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad); - - break; case IMB_AUTH_HMAC_SHA_384: - /* compute ipad hash */ - NOSIMD_MEMSET(buf, 0x36, IMB_SHA_384_BLOCK_SIZE); - for (i = 0; i < IMB_SHA_384_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad); - - /* compute opad hash */ - NOSIMD_MEMSET(buf, 0x5c, IMB_SHA_384_BLOCK_SIZE); - for (i = 0; i < IMB_SHA_384_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad); - - break; case IMB_AUTH_HMAC_SHA_512: - /* compute ipad hash */ - NOSIMD_MEMSET(buf, 0x36, IMB_SHA_512_BLOCK_SIZE); - for (i = 0; i < IMB_SHA_512_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad); - - /* compute opad hash */ - NOSIMD_MEMSET(buf, 0x5c, IMB_SHA_512_BLOCK_SIZE); - for (i = 0; i < IMB_SHA_512_BLOCK_SIZE; i++) - buf[i] ^= auth_key[i]; - IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad); - - break; case IMB_AUTH_MD5: - /* compute ipad hash */ - NOSIMD_MEMSET(buf, 0x36, 64); - for (i = 0; i < 64; i++) - buf[i] ^= auth_key[i]; - IMB_MD5_ONE_BLOCK(mb_mgr, buf, ipad); - - /* compute opad hash */ - NOSIMD_MEMSET(buf, 0x5c, 64); - for (i = 0; i < 64; i++) - buf[i] ^= auth_key[i]; - IMB_MD5_ONE_BLOCK(mb_mgr, buf, opad); - + imb_hmac_ipad_opad(mb_mgr, params->hash_alg, auth_key, + MAX_KEY_SIZE, ipad, opad); break; +#endif /* __aarch64__ */ case IMB_AUTH_ZUC_EIA3_BITLEN: case IMB_AUTH_ZUC256_EIA3_BITLEN: case IMB_AUTH_SNOW3G_UIA2_BITLEN: +#ifndef __aarch64__ case IMB_AUTH_KASUMI_UIA1: - NOSIMD_MEMCPY(k2, auth_key, sizeof(keys->k2)); +#endif /* __aarch64__ */ + nosimd_memcpy(k2, auth_key, sizeof(keys->k2)); break; +#ifndef __aarch64__ case IMB_AUTH_AES_GMAC_128: IMB_AES128_GCM_PRE(mb_mgr, auth_key, gdata_key); break; @@ -1518,8 +1488,13 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, /* No operation needed */ break; case IMB_AUTH_POLY1305: - NOSIMD_MEMCPY(k1_expanded, auth_key, 32); + nosimd_memcpy(k1_expanded, auth_key, 32); break; +#else /* __aarch64__ */ + case IMB_AUTH_NULL: + /* No operation needed */ + break; +#endif /* __aarch64__ */ default: fprintf(stderr, "Unsupported hash algorithm %u, line %d\n", (unsigned) params->hash_alg, __LINE__); @@ -1587,33 +1562,29 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_CIPHER_DOCSIS_DES: des_key_schedule((uint64_t *) enc_keys, ciph_key); break; +#endif /* __aarch64__ */ case IMB_CIPHER_SNOW3G_UEA2_BITLEN: +#ifndef __aarch64__ case IMB_CIPHER_KASUMI_UEA1_BITLEN: - NOSIMD_MEMCPY(k2, ciph_key, 16); +#endif /* __aarch64__ */ + nosimd_memcpy(k2, ciph_key, 16); break; case IMB_CIPHER_ZUC_EEA3: +#ifndef __aarch64__ case IMB_CIPHER_CHACHA20: case IMB_CIPHER_CHACHA20_POLY1305: case IMB_CIPHER_CHACHA20_POLY1305_SGL: case IMB_CIPHER_SNOW_V: case IMB_CIPHER_SNOW_V_AEAD: +#endif /* __aarch64__ */ /* Use of: - * NOSIMD_MEMCPY(k2, ciph_key, 32); + * nosimd_memcpy(k2, ciph_key, 32); * leaves sensitive data on the stack. * Copying data in 16 byte chunks instead. */ - NOSIMD_MEMCPY(k2, ciph_key, 16); - NOSIMD_MEMCPY(k2 + 16, ciph_key + 16, 16); - break; -#else - case IMB_CIPHER_SNOW3G_UEA2_BITLEN: - memcpy(k2, ciph_key, 16); - break; - case IMB_CIPHER_ZUC_EEA3: - memcpy(k2, ciph_key, 16); - memcpy(k2 + 16, ciph_key + 16, 16); + nosimd_memcpy(k2, ciph_key, 16); + nosimd_memcpy(k2 + 16, ciph_key + 16, 16); break; -#endif case IMB_CIPHER_NULL: /* No operation needed */ break; @@ -1669,7 +1640,7 @@ modify_docsis_crc32_test_buf(uint8_t *test_buf, if (buf_size >= (IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE + IMB_DOCSIS_CRC32_TAG_SIZE)) { /* Set plaintext CRC32 in the test buffer */ - NOSIMD_MEMCPY(&test_buf[buf_size - IMB_DOCSIS_CRC32_TAG_SIZE], + nosimd_memcpy(&test_buf[buf_size - IMB_DOCSIS_CRC32_TAG_SIZE], job->auth_tag_output, IMB_DOCSIS_CRC32_TAG_SIZE); } } @@ -1691,9 +1662,9 @@ perform_safe_checks(IMB_MGR *mgr, const IMB_ARCH arch, const char *dir) dump_gps(); switch (arch) { -#ifdef __x86_64__ - case IMB_ARCH_NOAESNI: +#ifndef __aarch64__ case IMB_ARCH_SSE: + case IMB_ARCH_NOAESNI: dump_xmms_sse(); simd_size = XMM_MEM_SIZE; break; @@ -1709,9 +1680,7 @@ perform_safe_checks(IMB_MGR *mgr, const IMB_ARCH arch, const char *dir) dump_zmms(); simd_size = ZMM_MEM_SIZE; break; -#endif - -#ifdef __aarch64__ +#else /* __aarch64__ */ case IMB_ARCH_NOAESNI: case IMB_ARCH_AARCH64: dump_simd_regs(); @@ -1776,8 +1745,6 @@ perform_safe_checks(IMB_MGR *mgr, const IMB_ARCH arch, const char *dir) }; void *ooo_mgr_p = *ooo_ptr; - if (ooo_mgr_p == NULL) continue; - if (search_patterns(ooo_mgr_p, get_ooo_mgr_size(ooo_mgr_p, i)) == 0) { fprintf(stderr, @@ -1831,8 +1798,8 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, generate_random_buf(auth_iv, MAX_IV_SIZE); generate_random_buf(aad, MAX_AAD_SIZE); if (safe_check) { - NOSIMD_MEMSET(ciph_key, pattern_cipher_key, MAX_KEY_SIZE); - NOSIMD_MEMSET(auth_key, pattern_auth_key, MAX_KEY_SIZE); + nosimd_memset(ciph_key, pattern_cipher_key, MAX_KEY_SIZE); + nosimd_memset(auth_key, pattern_auth_key, MAX_KEY_SIZE); } else { generate_random_buf(ciph_key, MAX_KEY_SIZE); generate_random_buf(auth_key, MAX_KEY_SIZE); @@ -1909,7 +1876,7 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, } if (safe_check) - NOSIMD_MEMSET(test_buf[i], pattern_plain_text, + nosimd_memset(test_buf[i], pattern_plain_text, buf_sizes[i]); else generate_random_buf(test_buf[i], buf_sizes[i]); @@ -2019,7 +1986,7 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, * Encrypt + generate digest from encrypted message * using architecture under test */ - NOSIMD_MEMCPY(src_dst_buf[i], test_buf[i], buf_sizes[i]); + nosimd_memcpy(src_dst_buf[i], test_buf[i], buf_sizes[i]); if (fill_job(job, params, src_dst_buf[i], in_digest[i], aad, buf_sizes[i], tag_size, IMB_DIR_ENCRYPT, enc_keys, cipher_iv, auth_iv, i, next_iv) < 0) @@ -2031,12 +1998,14 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Clear scratch registers before submitting job to prevent * other functions from storing sensitive data in stack */ job = IMB_SUBMIT_JOB(enc_mb_mgr); -#ifdef __x86_64__ + +#ifndef __aarch64__ avx_sse_check("enc-submit", (unsigned) params->hash_alg, (unsigned) params->cipher_mode); #endif + if (job) { - unsigned idx = (unsigned)((uintptr_t) job->user_data); + const unsigned idx = (unsigned)((uintptr_t) job->user_data); if (job->status != IMB_STATUS_COMPLETED) { int errc = imb_get_errno(enc_mb_mgr); @@ -2049,7 +2018,10 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, goto exit; } if (idx != num_processed_jobs) { - fprintf(stderr, "job returned out of order\n"); + fprintf(stderr, + "enc-submit job returned out of order, " + "received %u, expected %u\n", + idx, num_processed_jobs); goto exit; } num_processed_jobs++; @@ -2069,12 +2041,14 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Flush rest of the jobs, if there are outstanding jobs */ while (num_processed_jobs != num_jobs) { job = IMB_FLUSH_JOB(enc_mb_mgr); -#ifdef __x86_64__ + +#ifndef __aarch64__ avx_sse_check("enc-flush", (unsigned) params->hash_alg, (unsigned) params->cipher_mode); #endif + while (job != NULL) { - unsigned idx = (unsigned)((uintptr_t) job->user_data); + const unsigned idx = (unsigned)((uintptr_t) job->user_data); if (job->status != IMB_STATUS_COMPLETED) { int errc = imb_get_errno(enc_mb_mgr); @@ -2087,7 +2061,10 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, goto exit; } if (idx != num_processed_jobs) { - fprintf(stderr, "job returned out of order\n"); + fprintf(stderr, + "enc-flush job returned out of order, " + "received %u, expected %u\n", + idx, num_processed_jobs); goto exit; } num_processed_jobs++; @@ -2157,12 +2134,14 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Clear scratch registers before submitting job to prevent * other functions from storing sensitive data in stack */ job = IMB_SUBMIT_JOB(dec_mb_mgr); -#ifdef __x86_64__ + +#ifndef __aarch64__ avx_sse_check("dec-submit", (unsigned) params->hash_alg, (unsigned) params->cipher_mode); #endif + if (job != NULL) { - unsigned idx = (unsigned)((uintptr_t) job->user_data); + const unsigned idx = (unsigned)((uintptr_t) job->user_data); if (job->status != IMB_STATUS_COMPLETED) { int errc = imb_get_errno(dec_mb_mgr); @@ -2176,7 +2155,10 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, } if (idx != num_processed_jobs) { - fprintf(stderr, "job returned out of order\n"); + fprintf(stderr, + "dec-submit job returned out of order, " + "received %u, expected %u\n", + idx, num_processed_jobs); goto exit; } num_processed_jobs++; @@ -2186,12 +2168,14 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Flush rest of the jobs, if there are outstanding jobs */ while (num_processed_jobs != num_jobs) { job = IMB_FLUSH_JOB(dec_mb_mgr); -#ifdef __x86_64__ + +#ifndef __aarch64__ avx_sse_check("dec-flush", (unsigned) params->hash_alg, (unsigned) params->cipher_mode); #endif + while (job != NULL) { - unsigned idx = (unsigned)((uintptr_t) job->user_data); + const unsigned idx = (unsigned)((uintptr_t) job->user_data); if (job->status != IMB_STATUS_COMPLETED) { int errc = imb_get_errno(enc_mb_mgr); @@ -2204,7 +2188,10 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, goto exit; } if (idx != num_processed_jobs) { - fprintf(stderr, "job returned out of order\n"); + fprintf(stderr, + "dec-flush job returned out of order, " + "received %u, expected %u\n", + idx, num_processed_jobs); goto exit; } num_processed_jobs++; @@ -2340,9 +2327,8 @@ test_single(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, for (i = 4; i <= 16; i += 2) tag_sizes[num_tag_sizes++] = i; /* If ZUC-EIA3-256, test all tag sizes supported (4,8,16) */ - /* Todo, only 4 bytes tag is supported on arm arch */ } else if (params->hash_alg == IMB_AUTH_ZUC256_EIA3_BITLEN) { - for (i = 4; i <= 4; i *= 2) + for (i = 4; i <= 16; i *= 2) tag_sizes[num_tag_sizes++] = i; } else { tag_sizes[0] = auth_tag_len_bytes[params->hash_alg - 1]; @@ -2506,8 +2492,8 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, switch (enc_arch) { #ifndef __aarch64__ - case IMB_ARCH_NOAESNI: case IMB_ARCH_SSE: + case IMB_ARCH_NOAESNI: init_mb_mgr_sse(enc_mgr); break; case IMB_ARCH_AVX: @@ -2564,8 +2550,8 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, switch (dec_arch) { #ifndef __aarch64__ - case IMB_ARCH_NOAESNI: case IMB_ARCH_SSE: + case IMB_ARCH_NOAESNI: init_mb_mgr_sse(dec_mgr); break; case IMB_ARCH_AVX: @@ -2622,6 +2608,7 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, /* Skip IMB_CIPHER_CUSTOM */ if (c_mode == IMB_CIPHER_CUSTOM) continue; + #ifdef __aarch64__ if ((c_mode != IMB_CIPHER_NULL) && (c_mode != IMB_CIPHER_SNOW3G_UEA2_BITLEN) && @@ -2636,6 +2623,7 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, /* Skip IMB_AUTH_CUSTOM */ if (hash_alg == IMB_AUTH_CUSTOM) continue; + #ifdef __aarch64__ if ((hash_alg != IMB_AUTH_NULL) && (hash_alg != IMB_AUTH_SNOW3G_UIA2_BITLEN) && @@ -3023,7 +3011,7 @@ int main(int argc, char *argv[]) * Disable all the other architectures * and enable only the specified */ - NOSIMD_MEMSET(enc_archs, 0, sizeof(enc_archs)); + nosimd_memset(enc_archs, 0, sizeof(enc_archs)); enc_archs[values->arch_type] = 1; i++; } else if (strcmp(argv[i], "--dec-arch") == 0) { @@ -3038,7 +3026,7 @@ int main(int argc, char *argv[]) * Disable all the other architectures * and enable only the specified */ - NOSIMD_MEMSET(dec_archs, 0, sizeof(dec_archs)); + nosimd_memset(dec_archs, 0, sizeof(dec_archs)); dec_archs[values->arch_type] = 1; i++; } else if (strcmp(argv[i], "--cipher-algo") == 0) { @@ -3143,7 +3131,7 @@ int main(int argc, char *argv[]) } } else if (strcmp(argv[i], "--imix") == 0) { imix_enabled = 1; -#ifdef __x86_64__ +#ifndef __aarch64__ } else if (strcmp(argv[i], "--avx-sse") == 0) { is_avx_sse_check_possible = avx_sse_detectability(); if (!is_avx_sse_check_possible) diff --git a/test/misc.asm b/test/xvalid-app/misc.asm similarity index 88% rename from test/misc.asm rename to test/xvalid-app/misc.asm index f5e5b54fd416a7d91f131c0056ae63f570b6c294..eb0358a4bddab96b5ef2a44dfa3834f5b16bc32c 100644 --- a/test/misc.asm +++ b/test/xvalid-app/misc.asm @@ -1,5 +1,5 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright(c) 2019-2022, Intel Corporation All rights reserved. +; Copyright(c) 2019-2023, Intel Corporation All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -36,10 +36,15 @@ %define MKGLOBAL(name,type,scope) global name %+ : %+ type scope ;;; ABI function arguments -%define arg1 rdi -%define arg2 rsi -%define arg3 rdx -%define arg4 rcx +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx + +%define arg1d edi +%define arg2d esi +%define arg3d edx +%define arg4d ecx %endif %ifdef WIN_ABI @@ -50,10 +55,15 @@ %define MKGLOBAL(name,type,scope) global name ;;; ABI function arguments -%define arg1 rcx -%define arg2 rdx -%define arg3 r8 -%define arg4 r9 +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 + +%define arg1d ecx +%define arg2d edx +%define arg3d r8d +%define arg4d r9d %endif section .bss @@ -342,6 +352,34 @@ clr_scratch_zmms: ret +;; +;; Wrapper for CPUID opcode +;; +;; Parameters: +;; [in] leaf - CPUID leaf number (EAX) +;; [in] subleaf - CPUID sub-leaf number (ECX) +;; [out] out - registers structure to store results of CPUID into +;; +;; void misc_cpuid(const unsigned leaf, const unsigned subleaf, struct cpuid_regs *out) + +MKGLOBAL(misc_cpuid,function,internal) +misc_cpuid: + push rbx + + mov r11, arg3 ;; arg3 will get overwritten by cpuid on sysv + mov eax, arg1d + mov ecx, arg2d + + cpuid + + mov [r11 + 0*4], eax + mov [r11 + 1*4], ebx + mov [r11 + 2*4], ecx + mov [r11 + 3*4], edx + + pop rbx + ret + %ifdef LINUX section .note.GNU-stack noalloc noexec nowrite progbits %endif diff --git a/test/misc.h b/test/xvalid-app/misc.h similarity index 89% rename from test/misc.h rename to test/xvalid-app/misc.h index 42af339ca2e1519f897f6607010cd443b1adf37c..e3762b182aecff71b5d46697821c7f924edc726a 100644 --- a/test/misc.h +++ b/test/xvalid-app/misc.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright (c) 2019-2022, Intel Corporation + Copyright (c) 2019-2023, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -25,16 +25,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ -#ifdef __WIN32 -#include -#endif - #include #ifndef XVALIDAPP_MISC_H #define XVALIDAPP_MISC_H -#ifdef __x86_64__ +#ifndef __aarch64__ /* RAX, RBX, RCX, RDX, RDI, RSI, R8-R15 */ #define GP_MEM_SIZE 14*8 @@ -68,8 +64,6 @@ void *nosimd_memset(void *p, int c, size_t n); /* custom replacement for memcpy() */ void *nosimd_memcpy(void *dst, const void *src, size_t n); -#define NOSIMD_MEMCPY nosimd_memcpy -#define NOSIMD_MEMSET nosimd_memset /* * Detects if SIMD registers are in the state that * can cause AVX-SSE transition penalty @@ -97,34 +91,8 @@ struct misc_cpuid_regs { * @param subleaf[in] CPUID sub-leaf number (ECX) * @param out[out] registers structure to store results of CPUID into */ -static void -misc_cpuid(const unsigned leaf, const unsigned subleaf, - struct misc_cpuid_regs *out) -{ -#ifdef _WIN32 - /* Windows */ - int regs[4]; - - __cpuidex(regs, leaf, subleaf); - out->eax = regs[0]; - out->ebx = regs[1]; - out->ecx = regs[2]; - out->edx = regs[3]; -#else - /* Linux */ - asm volatile("mov %4, %%eax\n\t" - "mov %5, %%ecx\n\t" - "cpuid\n\t" - "mov %%eax, %0\n\t" - "mov %%ebx, %1\n\t" - "mov %%ecx, %2\n\t" - "mov %%edx, %3\n\t" - : "=g" (out->eax), "=g" (out->ebx), "=g" (out->ecx), - "=g" (out->edx) - : "g" (leaf), "g" (subleaf) - : "%eax", "%ebx", "%ecx", "%edx"); -#endif /* Linux */ -} +void misc_cpuid(const unsigned leaf, const unsigned subleaf, + struct misc_cpuid_regs *out); /** * @brief Detects if XGETBV instruction is available to use. @@ -321,11 +289,13 @@ static const char *misc_hash_alg_to_str(const IMB_HASH_ALG mode) snprintf(cb, sizeof(cb) - 1, "unknown<%u>", (unsigned) mode); return cb; } -#endif /* __x86_64__ */ -#ifdef __aarch64__ -#define NOSIMD_MEMCPY memcpy -#define NOSIMD_MEMSET memset +#else /* __aarch64__ */ + +void *nosimd_memcpy(void *str1, const void *str2, size_t n); + +void *nosimd_memset(void *str, int c, size_t n); + /* x0-x28 */ #define GP_MEM_SIZE 29*8 diff --git a/test/misc_aarch64.S b/test/xvalid-app/misc_aarch64.S similarity index 89% rename from test/misc_aarch64.S rename to test/xvalid-app/misc_aarch64.S index a1357fae93cef0689cf4943edf956c49e0a32b9b..27882c2467e8fe66ba98c7f9e29ed5d583f92f87 100644 --- a/test/misc_aarch64.S +++ b/test/xvalid-app/misc_aarch64.S @@ -28,6 +28,35 @@ **********************************************************************/ .arch armv8-a+sve +.global nosimd_memcpy +.type nosimd_memcpy,%function +.align 5 +nosimd_memcpy: + mov x10, #0 +cpy_begin: + cmp x10, x2 + b.ge cpy_end + ldrb w9, [x1, x10] + strb w9, [x0, x10] + add x10, x10, 1 + b cpy_begin +cpy_end: + ret + +.global nosimd_memset +.type nosimd_memset,%function +.align 5 +nosimd_memset: + mov x10, #0 +set_begin: + cmp x10, x2 + b.ge set_end + strb w1, [x0, x10] + add x10, x10, 1 + b set_begin +set_end: + ret + .global dump_gps .type dump_gps,%function .align 5 diff --git a/test/xvalid-app/win_x64.mak b/test/xvalid-app/win_x64.mak new file mode 100644 index 0000000000000000000000000000000000000000..8672c762e0b277442a3396084ba7f201d7622d2b --- /dev/null +++ b/test/xvalid-app/win_x64.mak @@ -0,0 +1,62 @@ +# +# Copyright (c) 2023, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP = imb-xvalid + +include ..\common\win_x64_common.mk + +AS = nasm +AFLAGS = -Werror -fwin64 -Xvc -DWIN_ABI + +XVALID_OBJS = ipsec_xvalid.obj misc.obj utils.obj +XVALID_LFLAGS = /out:$(APP).exe $(DLFLAGS) + +all: $(APP).exe tests.dep + +$(APP).exe: $(XVALID_OBJS) $(IPSECLIB) + $(LNK) $(XVALID_LFLAGS) $(XVALID_OBJS) $(IPSECLIB) + +tests.dep: $(TEST_OBJS) $(XVALID_OBJS) + @type *.obj.dep > $@ 2> nul + +.c.obj: + $(CC) /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{..\common\}.c.obj: + $(CC) /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +.asm.obj: + $(AS) -MD $@.dep -o $@ $(AFLAGS) $< + +clean: + del /q tests.dep *.obj.dep $(XVALID_OBJS) $(APP).* + +!if exist(tests.dep) +!include tests.dep +!endif diff --git a/win_x64.mak b/win_x64.mak index 8fac25413477fe29a1ec1b814dd6ab0b401fd281..5f52f109d8c9daec1e4f458aac7c9410ff1a522e 100644 --- a/win_x64.mak +++ b/win_x64.mak @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022, Intel Corporation +# Copyright (c) 2020-2023, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -28,13 +28,11 @@ all: cd lib & $(MAKE) /f win_x64.mak cd test & $(MAKE) /f win_x64.mak - cd test\wycheproof & $(MAKE) /f win_x64.mak cd perf & $(MAKE) /f win_x64.mak clean: cd lib & $(MAKE) /f win_x64.mak clean cd test & $(MAKE) /f win_x64.mak clean - cd test\wycheproof & $(MAKE) /f win_x64.mak clean cd perf & $(MAKE) /f win_x64.mak clean install: